sys/sun4v/sun4v/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  * $FreeBSD$
  41  */
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ktr.h"
  45 #include "opt_ktrace.h"
  46
  47 #include <sys/param.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/bus.h>
  51 #include <sys/interrupt.h>
  52 #include <sys/ktr.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/systm.h>
  56 #include <sys/pioctl.h>
  57 #include <sys/ptrace.h>
  58 #include <sys/proc.h>
  59 #include <sys/smp.h>
  60 #include <sys/signalvar.h>
  61 #include <sys/syscall.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/vmmeter.h>
  65 #ifdef KTRACE
  66 #include <sys/uio.h>
  67 #include <sys/ktrace.h>
  68 #endif
  69
  70 #include <dev/ofw/openfirm.h>
  71
  72 #include <vm/vm.h>
  73 #include <vm/pmap.h>
  74 #include <vm/vm_extern.h>
  75 #include <vm/vm_param.h>
  76 #include <vm/vm_kern.h>
  77 #include <vm/vm_map.h>
  78 #include <vm/vm_page.h>
  79
  80 #include <machine/clock.h>
  81 #include <machine/cpu.h>
  82 #include <machine/frame.h>
  83 #include <machine/intr_machdep.h>
  84 #include <machine/pcb.h>
  85 #include <machine/smp.h>
  86 #include <machine/trap.h>
  87 #include <machine/tstate.h>
  88 #include <machine/tte.h>
  89 #include <machine/tlb.h>
  90 #include <machine/tsb.h>
  91 #include <machine/watch.h>
  92 #include <machine/wstate.h>
  93
  94 #include <machine/md_var.h>
  95 #include <machine/hypervisorvar.h>
  96
  97 #include <security/audit/audit.h>
  98
  99 void trap(struct trapframe *tf, int64_t type, uint64_t data);
 100 void syscall(struct trapframe *tf);
 101
 102 extern vm_paddr_t mmu_fault_status_area;
 103
 104 static int trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data);
 105
 106 extern char copy_fault[];
 107 extern char copy_nofault_begin[];
 108 extern char copy_nofault_end[];
 109
 110 extern char fs_fault[];
 111 extern char fs_nofault_begin[];
 112 extern char fs_nofault_end[];
 113 extern char fs_nofault_intr_begin[];
 114 extern char fs_nofault_intr_end[];
 115
 116 extern char fas_fault[];
 117 extern char fas_nofault_begin[];
 118 extern char fas_nofault_end[];
 119
 120 extern char *syscallnames[];
 121
 122 const char *const trap_msg[] = {
 123         "reserved",
 124         "instruction access exception",
 125         "instruction access error",
 126         "instruction access protection",
 127         "illtrap instruction",
 128         "illegal instruction",
 129         "privileged opcode",
 130         "floating point disabled",
 131         "floating point exception ieee 754",
 132         "floating point exception other",
 133         "tag overflow",
 134         "division by zero",
 135         "data access exception",
 136         "data access error",
 137         "data access protection",
 138         "memory address not aligned",
 139         "privileged action",
 140         "async data error",
 141         "trap instruction 16",
 142         "trap instruction 17",
 143         "trap instruction 18",
 144         "trap instruction 19",
 145         "trap instruction 20",
 146         "trap instruction 21",
 147         "trap instruction 22",
 148         "trap instruction 23",
 149         "trap instruction 24",
 150         "trap instruction 25",
 151         "trap instruction 26",
 152         "trap instruction 27",
 153         "trap instruction 28",
 154         "trap instruction 29",
 155         "trap instruction 30",
 156         "trap instruction 31",
 157         "fast instruction access mmu miss",
 158         "fast data access mmu miss",
 159         "interrupt",
 160         "physical address watchpoint",
 161         "virtual address watchpoint",
 162         "corrected ecc error",
 163         "spill",
 164         "fill",
 165         "fill",
 166         "breakpoint",
 167         "clean window",
 168         "range check",
 169         "fix alignment",
 170         "integer overflow",
 171         "syscall",
 172         "restore physical watchpoint",
 173         "restore virtual watchpoint",
 174         "kernel stack fault",
 175         "resumable error",
 176         "nonresumable error"
 177 };
 178
 179 const int trap_sig[] = {
 180         SIGILL,                 /* reserved */
 181         SIGILL,                 /* instruction access exception */
 182         SIGILL,                 /* instruction access error */
 183         SIGILL,                 /* instruction access protection */
 184         SIGILL,                 /* illtrap instruction */
 185         SIGILL,                 /* illegal instruction */
 186         SIGBUS,                 /* privileged opcode */
 187         SIGFPE,                 /* floating point disabled */
 188         SIGFPE,                 /* floating point exception ieee 754 */
 189         SIGFPE,                 /* floating point exception other */
 190         SIGEMT,                 /* tag overflow */
 191         SIGFPE,                 /* division by zero */
 192         SIGILL,                 /* data access exception */
 193         SIGILL,                 /* data access error */
 194         SIGBUS,                 /* data access protection */
 195         SIGBUS,                 /* memory address not aligned */
 196         SIGBUS,                 /* privileged action */
 197         SIGBUS,                 /* async data error */
 198         SIGILL,                 /* trap instruction 16 */
 199         SIGILL,                 /* trap instruction 17 */
 200         SIGILL,                 /* trap instruction 18 */
 201         SIGILL,                 /* trap instruction 19 */
 202         SIGILL,                 /* trap instruction 20 */
 203         SIGILL,                 /* trap instruction 21 */
 204         SIGILL,                 /* trap instruction 22 */
 205         SIGILL,                 /* trap instruction 23 */
 206         SIGILL,                 /* trap instruction 24 */
 207         SIGILL,                 /* trap instruction 25 */
 208         SIGILL,                 /* trap instruction 26 */
 209         SIGILL,                 /* trap instruction 27 */
 210         SIGILL,                 /* trap instruction 28 */
 211         SIGILL,                 /* trap instruction 29 */
 212         SIGILL,                 /* trap instruction 30 */
 213         SIGILL,                 /* trap instruction 31 */
 214         SIGFPE,                 /* floating point error */
 215         SIGSEGV,                /* fast data access mmu miss */
 216         -1,                     /* interrupt */
 217         -1,                     /* physical address watchpoint */
 218         -1,                     /* virtual address watchpoint */
 219         -1,                     /* corrected ecc error */
 220         SIGILL,                 /* spill */
 221         SIGILL,                 /* fill */
 222         SIGILL,                 /* fill */
 223         SIGTRAP,                /* breakpoint */
 224         SIGILL,                 /* clean window */
 225         SIGILL,                 /* range check */
 226         SIGILL,                 /* fix alignment */
 227         SIGILL,                 /* integer overflow */
 228         SIGSYS,                 /* syscall */
 229         -1,                     /* restore physical watchpoint */
 230         -1,                     /* restore virtual watchpoint */
 231         -1,                     /* kernel stack fault */
 232 };
 233
 234 CTASSERT(sizeof(struct trapframe) == 256);
 235
 236 int debugger_on_signal = 0;
 237 #ifdef DEBUG
 238 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 239     &debugger_on_signal, 0, "");
 240 #endif
 241
 242 /*
 243  * This interface allows the client to safely take over the %tba by
 244  * the prom's service. The prom will take care of the quiescence of
 245  * interrupts and handle any pending soft interrupts.
 246  * This call also sets the MMU fault status area for the CPU.
 247  */
 248 void
 249 set_mmfsa_traptable(void *tba_addr, uint64_t mmfsa_ra)
 250 {
 251         static struct {
 252         cell_t name;
 253                 cell_t nargs;
 254                 cell_t nreturns;
 255                 cell_t tba_addr;
 256                 cell_t mmfsa_ra;
 257         } args = {
 258                 (cell_t)"SUNW,set-trap-table",
 259                 2,
 260         };
 261
 262         args.tba_addr = (cell_t)tba_addr;
 263         args.mmfsa_ra = mmfsa_ra;
 264         ofw_entry(&args);
 265 }
 266
 267 void
 268 trap_init(void)
 269 {
 270         vm_paddr_t mmfsa;
 271
 272         mmfsa = mmu_fault_status_area + (MMFSA_SIZE*curcpu);
 273
 274         set_wstate(WSTATE_KERN);
 275         set_mmfsa_scratchpad(mmfsa);
 276
 277         init_mondo_queue();
 278         set_mmfsa_traptable(&tl0_base, mmfsa);
 279 }
 280
 281 void
 282 trap(struct trapframe *tf, int64_t type, uint64_t data)
 283 {
 284         struct thread *td;
 285         struct proc *p;
 286         int error, sig, ctx;
 287         uint64_t trapno;
 288         register_t addr;
 289         ksiginfo_t ksi;
 290
 291         td = curthread;
 292
 293         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 294             trap_msg[trapno],
 295             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 296
 297         PCPU_INC(cnt.v_trap);
 298
 299         trapno = (type & TRAP_MASK);
 300         ctx = (type >> TRAP_CTX_SHIFT);
 301
 302         if (((tf->tf_tstate & TSTATE_PRIV) == 0) || (ctx != 0)) {
 303                 KASSERT(td != NULL, ("trap: curthread NULL"));
 304                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 305
 306                 p = td->td_proc;
 307                 td->td_pticks = 0;
 308                 td->td_frame = tf;
 309                 addr = tf->tf_tpc;
 310                 if (td->td_ucred != p->p_ucred)
 311                         cred_update_thread(td);
 312
 313                 switch (trapno) {
 314                 case T_DATA_MISS:
 315                 case T_DATA_PROTECTION:
 316                         addr = TLB_TAR_VA(data);
 317                 case T_INSTRUCTION_MISS:
 318                         sig = trap_pfault(td, tf, trapno, data);
 319                         break;
 320                 case T_FILL:
 321                         sig = rwindow_load(td, tf, 2);
 322                         break;
 323                 case T_FILL_RET:
 324                         sig = rwindow_load(td, tf, 1);
 325                         break;
 326                 case T_SPILL:
 327                         sig = rwindow_save(td);
 328                         break;
 329                 case T_DATA_EXCEPTION:
 330                 case T_DATA_ERROR:
 331                 case T_MEM_ADDRESS_NOT_ALIGNED:
 332                         printf("bad trap trapno=%ld data=0x%lx pc=0x%lx\n",
 333                                trapno, data, tf->tf_tpc);
 334                         if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 335                             tf->tf_tpc <= (u_long)copy_nofault_end) {
 336                                 tf->tf_tpc = (u_long)copy_fault;
 337                                 tf->tf_tnpc = tf->tf_tpc + 4;
 338                                 sig = 0;
 339                                 break;
 340                         }
 341                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 342                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 343                                 tf->tf_tpc = (u_long)fs_fault;
 344                                 tf->tf_tnpc = tf->tf_tpc + 4;
 345                                 sig = 0;
 346                                 break;
 347                         }
 348
 349                         addr = data;
 350                         sig = trap_sig[trapno];
 351                         break;
 352
 353                 default:
 354                         if (trapno < 0 || trapno >= T_MAX ||
 355                             trap_sig[trapno] == -1)
 356                                 panic("trap: bad trap type");
 357                         sig = trap_sig[trapno];
 358                         break;
 359                 }
 360
 361                 if (sig != 0) {
 362                         /* Translate fault for emulators. */
 363                         if (p->p_sysent->sv_transtrap != NULL) {
 364                                 sig = p->p_sysent->sv_transtrap(sig,
 365                                     trapno);
 366                         }
 367                         if (debugger_on_signal &&
 368                             (sig == 4 || sig == 10 || sig == 11))
 369                                 kdb_enter(KDB_WHY_TRAPSIG, "trapsig");
 370 #ifdef VERBOSE
 371                         if (sig == 4 || sig == 10 || sig == 11)
 372                                 printf("trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d sig=%d proc=%s\n",
 373                                        trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu, sig, curthread->td_name);
 374 #endif
 375                         /* XXX I've renumbered the traps to largely reflect what the hardware uses
 376                          * so this will need to be re-visited
 377                          */
 378                         ksiginfo_init_trap(&ksi);
 379                         ksi.ksi_signo = sig;
 380                         ksi.ksi_code = (int)trapno; /* XXX not POSIX */
 381                         ksi.ksi_addr = (void *)addr;
 382                         ksi.ksi_trapno = (int)trapno;
 383                         trapsignal(td, &ksi);
 384                 }
 385
 386                 userret(td, tf);
 387                 mtx_assert(&Giant, MA_NOTOWNED);
 388         } else {
 389                 KASSERT((type & T_KERNEL) != 0,
 390                     ("trap: kernel trap isn't - trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d\n",
 391                      trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu));
 392
 393                 if (kdb_active) {
 394                         kdb_reenter();
 395                         return;
 396                 }
 397
 398                 switch (trapno) {
 399                 case T_BREAKPOINT:
 400                 case T_KSTACK_FAULT:
 401                         error = (kdb_trap(trapno, 0, tf) == 0);
 402                         TF_DONE(tf);
 403                         break;
 404                 case T_DATA_MISS:
 405                 case T_DATA_PROTECTION:
 406                 case T_INSTRUCTION_MISS:
 407                         error = trap_pfault(td, tf, trapno, data);
 408                         break;
 409                 case T_DATA_EXCEPTION:
 410                         printf("data exception on 0x%lx at 0x%lx\n", data, tf->tf_tpc);
 411                         printf("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx\n", trapno,
 412                                trap_msg[trapno], data, tf->tf_tpc, tf->tf_tnpc);
 413                case T_ILLEGAL_INSTRUCTION:
 414                        if (tf->tf_tpc > KERNBASE) {
 415                                printf("illinstr: 0x%lx\n", tf->tf_tpc);
 416                                printf("illinstr: 0x%x\n", *((uint32_t *)tf->tf_tpc));
 417                        }
 418                 case T_DATA_ERROR:
 419                 case T_ALIGNMENT:
 420                         if (tf->tf_asi == ASI_AIUS) {
 421                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 422                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 423                                         tf->tf_tpc = (u_long)copy_fault;
 424                                         tf->tf_tnpc = tf->tf_tpc + 4;
 425                                         error = 0;
 426                                         break;
 427                                 }
 428
 429                                 printf("ASI_AIUS but bad tpc\n");
 430                         }
 431                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 432                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 433                                 tf->tf_tpc = (u_long)fs_fault;
 434                                 tf->tf_tnpc = tf->tf_tpc + 4;
 435                                 error = 0;
 436                                         break;
 437                         }
 438                         printf("asi=0x%lx\n", tf->tf_asi);
 439                         error = 1;
 440                         break;
 441                 default:
 442                         printf("unchecked trap 0x%lx asi=0x%lx\n", trapno, tf->tf_asi);
 443                         error = 1;
 444                         break;
 445                 }
 446
 447                 if (error != 0)
 448                         panic("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx error=%d asi=0x%lx",
 449                               trapno, trap_msg[trapno], data, tf->tf_tpc,
 450                               tf->tf_tnpc, error, tf->tf_asi);
 451         }
 452         CTR1(KTR_TRAP, "trap: td=%p return", td);
 453 }
 454
 455 static int
 456 trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data)
 457 {
 458         struct vmspace *vm;
 459         struct proc *p;
 460         vm_offset_t va;
 461         vm_prot_t prot;
 462         u_long ctx;
 463         int flags;
 464         int rv;
 465
 466         if (td == NULL)
 467                 return (-1);
 468
 469         p = td->td_proc;
 470
 471         rv = KERN_SUCCESS;
 472         ctx = TLB_TAR_CTX(data);
 473         type = type & ~T_KERNEL;
 474         va = TLB_TAR_VA(data);
 475
 476
 477         if (data > VM_MIN_DIRECT_ADDRESS)
 478                 printf("trap_pfault(type=%ld, data=0x%lx, tpc=0x%lx, ctx=0x%lx)\n",
 479                        type, data, tf->tf_tpc, ctx);
 480
 481 #if 0
 482         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 483             td, p->p_vmspace->vm_pmap.pm_context, va, ctx);
 484 #endif
 485         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 486         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 487         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 488
 489
 490         if (type == T_DATA_PROTECTION) {
 491                 prot = VM_PROT_WRITE;
 492                 flags = VM_FAULT_DIRTY;
 493         } else {
 494                 if (type == T_DATA_MISS)
 495                         prot = VM_PROT_READ;
 496                 else
 497                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 498                 flags = VM_FAULT_NORMAL;
 499         }
 500
 501         if (ctx != TLB_CTX_KERNEL) {
 502                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 503                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 504                      tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 505                         tf->tf_tpc = (u_long)fs_fault;
 506                         tf->tf_tnpc = tf->tf_tpc + 4;
 507                         return (0);
 508                 }
 509
 510                 /*
 511                  * This is a fault on non-kernel virtual memory.
 512                  */
 513                 vm = p->p_vmspace;
 514
 515                 /*
 516                  * Keep swapout from messing with us during this
 517                  * critical time.
 518                  */
 519                 PROC_LOCK(p);
 520                 ++p->p_lock;
 521                 PROC_UNLOCK(p);
 522
 523                 /* Fault in the user page. */
 524                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 525
 526                 /*
 527                  * Now the process can be swapped again.
 528                  */
 529                 PROC_LOCK(p);
 530                 --p->p_lock;
 531                 PROC_UNLOCK(p);
 532         } else {
 533                 /*
 534                  * This is a fault on kernel virtual memory.  Attempts to
 535                  * access kernel memory from user mode cause privileged
 536                  * action traps, not page fault.
 537                  */
 538                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 539                     ("trap_pfault: fault on nucleus context from user mode"));
 540
 541                 /*
 542                  * Don't have to worry about process locking or stacks in the
 543                  * kernel.
 544                  */
 545                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 546         }
 547
 548         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 549             td, va, rv);
 550         if (rv == KERN_SUCCESS)
 551                 return (0);
 552         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 553                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 554                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 555                         tf->tf_tpc = (u_long)fs_fault;
 556                         tf->tf_tnpc = tf->tf_tpc + 4;
 557                         return (0);
 558                 }
 559                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 560                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 561                         tf->tf_tpc = (u_long)copy_fault;
 562                         tf->tf_tnpc = tf->tf_tpc + 4;
 563                         return (0);
 564                 }
 565         }
 566         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 567 }
 568
 569 /* Maximum number of arguments that can be passed via the out registers. */
 570 #define REG_MAXARGS     6
 571
 572 /*
 573  * Syscall handler. The arguments to the syscall are passed in the o registers
 574  * by the caller, and are saved in the trap frame. The syscall number is passed
 575  * in %g1 (and also saved in the trap frame).
 576  */
 577 void
 578 syscall(struct trapframe *tf)
 579 {
 580         struct sysent *callp;
 581         struct thread *td;
 582         register_t args[8];
 583         register_t *argp;
 584         struct proc *p;
 585         u_long code;
 586         int reg;
 587         int regcnt;
 588         int narg;
 589         int error;
 590
 591         td = curthread;
 592         KASSERT(td != NULL, ("trap: curthread NULL"));
 593         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 594
 595         p = td->td_proc;
 596
 597         PCPU_INC(cnt.v_syscall);
 598
 599         td->td_pticks = 0;
 600         td->td_frame = tf;
 601         if (td->td_ucred != p->p_ucred)
 602                 cred_update_thread(td);
 603         code = tf->tf_global[1];
 604
 605         /*
 606          * For syscalls, we don't want to retry the faulting instruction
 607          * (usually), instead we need to advance one instruction.
 608          */
 609         td->td_pcb->pcb_tpc = tf->tf_tpc;
 610         TF_DONE(tf);
 611
 612         reg = 0;
 613         regcnt = REG_MAXARGS;
 614         if (p->p_sysent->sv_prepsyscall) {
 615                 /*
 616                  * The prep code is MP aware.
 617                  */
 618 #if 0
 619                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 620 #endif
 621         } else  if (code == SYS_syscall || code == SYS___syscall) {
 622                 code = tf->tf_out[reg++];
 623                 regcnt--;
 624         }
 625
 626         if (p->p_sysent->sv_mask)
 627                 code &= p->p_sysent->sv_mask;
 628
 629         if (code >= p->p_sysent->sv_size)
 630                 callp = &p->p_sysent->sv_table[0];
 631         else
 632                 callp = &p->p_sysent->sv_table[code];
 633
 634         narg = callp->sy_narg;
 635
 636         if (narg <= regcnt) {
 637                 argp = &tf->tf_out[reg];
 638                 error = 0;
 639         } else {
 640                 KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 641                     ("Too many syscall arguments!"));
 642                 argp = args;
 643                 bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 644                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 645                     offsetof(struct frame, fr_pad[6])),
 646                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 647         }
 648
 649         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 650             syscallnames[code], argp[0], argp[1], argp[2]);
 651
 652         /*
 653          * Try to run the syscall without the MP lock if the syscall
 654          * is MP safe.
 655          */
 656 #ifdef KTRACE
 657         if (KTRPOINT(td, KTR_SYSCALL))
 658                 ktrsyscall(code, narg, argp);
 659 #endif
 660         if (error == 0) {
 661                 td->td_retval[0] = 0;
 662                 td->td_retval[1] = 0;
 663
 664                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 665
 666                 PTRACESTOP_SC(p, td, S_PT_SCE);
 667
 668                 AUDIT_SYSCALL_ENTER(code, td);
 669                 error = (*callp->sy_call)(td, argp);
 670                 AUDIT_SYSCALL_EXIT(error, td);
 671
 672                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 673                     error, syscallnames[code], td->td_retval[0],
 674                     td->td_retval[1]);
 675         }
 676
 677         cpu_set_syscall_retval(td, error);
 678
 679         /*
 680          * Handle reschedule and other end-of-syscall issues
 681          */
 682         userret(td, tf);
 683
 684 #ifdef KTRACE
 685         if (KTRPOINT(td, KTR_SYSRET))
 686                 ktrsysret(code, error, td->td_retval[0]);
 687 #endif
 688         /*
 689          * This works because errno is findable through the
 690          * register set.  If we ever support an emulation where this
 691          * is not the case, this code will need to be revisited.
 692          */
 693         STOPEVENT(p, S_SCX, code);
 694
 695         PTRACESTOP_SC(p, td, S_PT_SCX);
 696
 697         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 698             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 699         mtx_assert(&Giant, MA_NOTOWNED);
 700 }