sys/sun4v/sun4v/trap.c

   1 /*-
   2  * Copyright (c) 2001, Jake Burkholder
   3  * Copyright (C) 1994, David Greenman
   4  * Copyright (c) 1990, 1993
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * the University of Utah, and William Jolitz.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by the University of
  21  *      California, Berkeley and its contributors.
  22  * 4. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  39  *      from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19
  40  * $FreeBSD$
  41  */
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ktr.h"
  45 #include "opt_ktrace.h"
  46
  47 #include <sys/param.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/bus.h>
  51 #include <sys/interrupt.h>
  52 #include <sys/ktr.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/systm.h>
  56 #include <sys/pioctl.h>
  57 #include <sys/ptrace.h>
  58 #include <sys/proc.h>
  59 #include <sys/smp.h>
  60 #include <sys/signalvar.h>
  61 #include <sys/syscall.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sysent.h>
  64 #include <sys/vmmeter.h>
  65 #ifdef KTRACE
  66 #include <sys/uio.h>
  67 #include <sys/ktrace.h>
  68 #endif
  69
  70 #include <dev/ofw/openfirm.h>
  71
  72 #include <vm/vm.h>
  73 #include <vm/pmap.h>
  74 #include <vm/vm_extern.h>
  75 #include <vm/vm_param.h>
  76 #include <vm/vm_kern.h>
  77 #include <vm/vm_map.h>
  78 #include <vm/vm_page.h>
  79
  80 #include <machine/clock.h>
  81 #include <machine/cpu.h>
  82 #include <machine/frame.h>
  83 #include <machine/intr_machdep.h>
  84 #include <machine/smp.h>
  85 #include <machine/trap.h>
  86 #include <machine/tstate.h>
  87 #include <machine/tte.h>
  88 #include <machine/tlb.h>
  89 #include <machine/tsb.h>
  90 #include <machine/watch.h>
  91 #include <machine/wstate.h>
  92
  93 #include <machine/md_var.h>
  94 #include <machine/hypervisorvar.h>
  95
  96 #include <security/audit/audit.h>
  97
  98 void trap(struct trapframe *tf, int64_t type, uint64_t data);
  99 void syscall(struct trapframe *tf);
 100
 101 extern vm_paddr_t mmu_fault_status_area;
 102
 103 static int trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data);
 104
 105 extern char copy_fault[];
 106 extern char copy_nofault_begin[];
 107 extern char copy_nofault_end[];
 108
 109 extern char fs_fault[];
 110 extern char fs_nofault_begin[];
 111 extern char fs_nofault_end[];
 112 extern char fs_nofault_intr_begin[];
 113 extern char fs_nofault_intr_end[];
 114
 115 extern char fas_fault[];
 116 extern char fas_nofault_begin[];
 117 extern char fas_nofault_end[];
 118
 119 extern char *syscallnames[];
 120
 121 const char *const trap_msg[] = {
 122         "reserved",
 123         "instruction access exception",
 124         "instruction access error",
 125         "instruction access protection",
 126         "illtrap instruction",
 127         "illegal instruction",
 128         "privileged opcode",
 129         "floating point disabled",
 130         "floating point exception ieee 754",
 131         "floating point exception other",
 132         "tag overflow",
 133         "division by zero",
 134         "data access exception",
 135         "data access error",
 136         "data access protection",
 137         "memory address not aligned",
 138         "privileged action",
 139         "async data error",
 140         "trap instruction 16",
 141         "trap instruction 17",
 142         "trap instruction 18",
 143         "trap instruction 19",
 144         "trap instruction 20",
 145         "trap instruction 21",
 146         "trap instruction 22",
 147         "trap instruction 23",
 148         "trap instruction 24",
 149         "trap instruction 25",
 150         "trap instruction 26",
 151         "trap instruction 27",
 152         "trap instruction 28",
 153         "trap instruction 29",
 154         "trap instruction 30",
 155         "trap instruction 31",
 156         "fast instruction access mmu miss",
 157         "fast data access mmu miss",
 158         "interrupt",
 159         "physical address watchpoint",
 160         "virtual address watchpoint",
 161         "corrected ecc error",
 162         "spill",
 163         "fill",
 164         "fill",
 165         "breakpoint",
 166         "clean window",
 167         "range check",
 168         "fix alignment",
 169         "integer overflow",
 170         "syscall",
 171         "restore physical watchpoint",
 172         "restore virtual watchpoint",
 173         "kernel stack fault",
 174         "resumable error",
 175         "nonresumable error"
 176 };
 177
 178 const int trap_sig[] = {
 179         SIGILL,                 /* reserved */
 180         SIGILL,                 /* instruction access exception */
 181         SIGILL,                 /* instruction access error */
 182         SIGILL,                 /* instruction access protection */
 183         SIGILL,                 /* illtrap instruction */
 184         SIGILL,                 /* illegal instruction */
 185         SIGBUS,                 /* privileged opcode */
 186         SIGFPE,                 /* floating point disabled */
 187         SIGFPE,                 /* floating point exception ieee 754 */
 188         SIGFPE,                 /* floating point exception other */
 189         SIGEMT,                 /* tag overflow */
 190         SIGFPE,                 /* division by zero */
 191         SIGILL,                 /* data access exception */
 192         SIGILL,                 /* data access error */
 193         SIGBUS,                 /* data access protection */
 194         SIGBUS,                 /* memory address not aligned */
 195         SIGBUS,                 /* privileged action */
 196         SIGBUS,                 /* async data error */
 197         SIGILL,                 /* trap instruction 16 */
 198         SIGILL,                 /* trap instruction 17 */
 199         SIGILL,                 /* trap instruction 18 */
 200         SIGILL,                 /* trap instruction 19 */
 201         SIGILL,                 /* trap instruction 20 */
 202         SIGILL,                 /* trap instruction 21 */
 203         SIGILL,                 /* trap instruction 22 */
 204         SIGILL,                 /* trap instruction 23 */
 205         SIGILL,                 /* trap instruction 24 */
 206         SIGILL,                 /* trap instruction 25 */
 207         SIGILL,                 /* trap instruction 26 */
 208         SIGILL,                 /* trap instruction 27 */
 209         SIGILL,                 /* trap instruction 28 */
 210         SIGILL,                 /* trap instruction 29 */
 211         SIGILL,                 /* trap instruction 30 */
 212         SIGILL,                 /* trap instruction 31 */
 213         SIGFPE,                 /* floating point error */
 214         SIGSEGV,                /* fast data access mmu miss */
 215         -1,                     /* interrupt */
 216         -1,                     /* physical address watchpoint */
 217         -1,                     /* virtual address watchpoint */
 218         -1,                     /* corrected ecc error */
 219         SIGILL,                 /* spill */
 220         SIGILL,                 /* fill */
 221         SIGILL,                 /* fill */
 222         SIGTRAP,                /* breakpoint */
 223         SIGILL,                 /* clean window */
 224         SIGILL,                 /* range check */
 225         SIGILL,                 /* fix alignment */
 226         SIGILL,                 /* integer overflow */
 227         SIGSYS,                 /* syscall */
 228         -1,                     /* restore physical watchpoint */
 229         -1,                     /* restore virtual watchpoint */
 230         -1,                     /* kernel stack fault */
 231 };
 232
 233 CTASSERT(sizeof(struct trapframe) == 256);
 234
 235 int debugger_on_signal = 0;
 236 #ifdef DEBUG
 237 SYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW,
 238     &debugger_on_signal, 0, "");
 239 #endif
 240
 241 /*
 242  * This interface allows the client to safely take over the %tba by
 243  * the prom's service. The prom will take care of the quiescence of
 244  * interrupts and handle any pending soft interrupts.
 245  * This call also sets the MMU fault status area for the CPU.
 246  */
 247 void
 248 set_mmfsa_traptable(void *tba_addr, uint64_t mmfsa_ra)
 249 {
 250         static struct {
 251         cell_t name;
 252                 cell_t nargs;
 253                 cell_t nreturns;
 254                 cell_t tba_addr;
 255                 cell_t mmfsa_ra;
 256         } args = {
 257                 (cell_t)"SUNW,set-trap-table",
 258                 2,
 259         };
 260
 261         args.tba_addr = (cell_t)tba_addr;
 262         args.mmfsa_ra = mmfsa_ra;
 263         openfirmware(&args);
 264 }
 265
 266 void
 267 trap_init(void)
 268 {
 269         vm_paddr_t mmfsa;
 270
 271         mmfsa = mmu_fault_status_area + (MMFSA_SIZE*curcpu);
 272
 273         set_wstate(WSTATE_KERN);
 274         set_mmfsa_scratchpad(mmfsa);
 275
 276         init_mondo_queue();
 277         set_mmfsa_traptable(&tl0_base, mmfsa);
 278 }
 279
 280 void
 281 trap(struct trapframe *tf, int64_t type, uint64_t data)
 282 {
 283         struct thread *td;
 284         struct proc *p;
 285         int error, sig, ctx;
 286         uint64_t trapno;
 287         register_t addr;
 288         ksiginfo_t ksi;
 289
 290         td = curthread;
 291
 292         CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td,
 293             trap_msg[trapno],
 294             (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 295
 296         PCPU_INC(cnt.v_trap);
 297
 298         trapno = (type & TRAP_MASK);
 299         ctx = (type >> TRAP_CTX_SHIFT);
 300
 301         if (((tf->tf_tstate & TSTATE_PRIV) == 0) || (ctx != 0)) {
 302                 KASSERT(td != NULL, ("trap: curthread NULL"));
 303                 KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 304
 305                 p = td->td_proc;
 306                 td->td_pticks = 0;
 307                 td->td_frame = tf;
 308                 addr = tf->tf_tpc;
 309                 if (td->td_ucred != p->p_ucred)
 310                         cred_update_thread(td);
 311
 312                 switch (trapno) {
 313                 case T_DATA_MISS:
 314                 case T_DATA_PROTECTION:
 315                         addr = TLB_TAR_VA(data);
 316                 case T_INSTRUCTION_MISS:
 317                         sig = trap_pfault(td, tf, trapno, data);
 318                         break;
 319                 case T_FILL:
 320                         sig = rwindow_load(td, tf, 2);
 321                         break;
 322                 case T_FILL_RET:
 323                         sig = rwindow_load(td, tf, 1);
 324                         break;
 325                 case T_SPILL:
 326                         sig = rwindow_save(td);
 327                         break;
 328                 case T_DATA_EXCEPTION:
 329                 case T_DATA_ERROR:
 330                 case T_MEM_ADDRESS_NOT_ALIGNED:
 331                         printf("bad trap trapno=%ld data=0x%lx pc=0x%lx\n",
 332                                trapno, data, tf->tf_tpc);
 333                         if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 334                             tf->tf_tpc <= (u_long)copy_nofault_end) {
 335                                 tf->tf_tpc = (u_long)copy_fault;
 336                                 tf->tf_tnpc = tf->tf_tpc + 4;
 337                                 sig = 0;
 338                                 break;
 339                         }
 340                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 341                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 342                                 tf->tf_tpc = (u_long)fs_fault;
 343                                 tf->tf_tnpc = tf->tf_tpc + 4;
 344                                 sig = 0;
 345                                 break;
 346                         }
 347
 348                         addr = data;
 349                         sig = trap_sig[trapno];
 350                         break;
 351
 352                 default:
 353                         if (trapno < 0 || trapno >= T_MAX ||
 354                             trap_sig[trapno] == -1)
 355                                 panic("trap: bad trap type");
 356                         sig = trap_sig[trapno];
 357                         break;
 358                 }
 359
 360                 if (sig != 0) {
 361                         /* Translate fault for emulators. */
 362                         if (p->p_sysent->sv_transtrap != NULL) {
 363                                 sig = p->p_sysent->sv_transtrap(sig,
 364                                     trapno);
 365                         }
 366                         if (debugger_on_signal &&
 367                             (sig == 4 || sig == 10 || sig == 11))
 368                                 kdb_enter_why(KDB_WHY_TRAPSIG, "trapsig");
 369 #ifdef VERBOSE
 370                         if (sig == 4 || sig == 10 || sig == 11)
 371                                 printf("trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d sig=%d proc=%s\n",
 372                                        trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu, sig, curthread->td_proc->p_comm);
 373 #endif
 374                         /* XXX I've renumbered the traps to largely reflect what the hardware uses
 375                          * so this will need to be re-visited
 376                          */
 377                         ksiginfo_init_trap(&ksi);
 378                         ksi.ksi_signo = sig;
 379                         ksi.ksi_code = (int)trapno; /* XXX not POSIX */
 380                         ksi.ksi_addr = (void *)addr;
 381                         ksi.ksi_trapno = (int)trapno;
 382                         trapsignal(td, &ksi);
 383                 }
 384
 385                 userret(td, tf);
 386                 mtx_assert(&Giant, MA_NOTOWNED);
 387         } else {
 388                 KASSERT((type & T_KERNEL) != 0,
 389                     ("trap: kernel trap isn't - trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d\n",
 390                      trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu));
 391
 392                 if (kdb_active) {
 393                         kdb_reenter();
 394                         return;
 395                 }
 396
 397                 switch (trapno) {
 398                 case T_BREAKPOINT:
 399                 case T_KSTACK_FAULT:
 400                         error = (kdb_trap(trapno, 0, tf) == 0);
 401                         TF_DONE(tf);
 402                         break;
 403                 case T_DATA_MISS:
 404                 case T_DATA_PROTECTION:
 405                 case T_INSTRUCTION_MISS:
 406                         error = trap_pfault(td, tf, trapno, data);
 407                         break;
 408                 case T_DATA_EXCEPTION:
 409                         printf("data exception on 0x%lx at 0x%lx\n", data, tf->tf_tpc);
 410                         printf("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx\n", trapno,
 411                                trap_msg[trapno], data, tf->tf_tpc, tf->tf_tnpc);
 412                case T_ILLEGAL_INSTRUCTION:
 413                        if (tf->tf_tpc > KERNBASE) {
 414                                printf("illinstr: 0x%lx\n", tf->tf_tpc);
 415                                printf("illinstr: 0x%x\n", *((uint32_t *)tf->tf_tpc));
 416                        }
 417                 case T_DATA_ERROR:
 418                 case T_ALIGNMENT:
 419                         if (tf->tf_asi == ASI_AIUS) {
 420                                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 421                                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 422                                         tf->tf_tpc = (u_long)copy_fault;
 423                                         tf->tf_tnpc = tf->tf_tpc + 4;
 424                                         error = 0;
 425                                         break;
 426                                 }
 427
 428                                 printf("ASI_AIUS but bad tpc\n");
 429                         }
 430                         if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 431                             tf->tf_tpc <= (u_long)fs_nofault_end) {
 432                                 tf->tf_tpc = (u_long)fs_fault;
 433                                 tf->tf_tnpc = tf->tf_tpc + 4;
 434                                 error = 0;
 435                                         break;
 436                         }
 437                         printf("asi=0x%lx\n", tf->tf_asi);
 438                         error = 1;
 439                         break;
 440                 default:
 441                         printf("unchecked trap 0x%lx asi=0x%lx\n", trapno, tf->tf_asi);
 442                         error = 1;
 443                         break;
 444                 }
 445
 446                 if (error != 0)
 447                         panic("trap: %ld=%s: 0x%lx at 0x%lx:0x%lx error=%d asi=0x%lx",
 448                               trapno, trap_msg[trapno], data, tf->tf_tpc,
 449                               tf->tf_tnpc, error, tf->tf_asi);
 450         }
 451         CTR1(KTR_TRAP, "trap: td=%p return", td);
 452 }
 453
 454 static int
 455 trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data)
 456 {
 457         struct vmspace *vm;
 458         struct proc *p;
 459         vm_offset_t va;
 460         vm_prot_t prot;
 461         u_long ctx;
 462         int flags;
 463         int rv;
 464
 465         if (td == NULL)
 466                 return (-1);
 467
 468         p = td->td_proc;
 469
 470         rv = KERN_SUCCESS;
 471         ctx = TLB_TAR_CTX(data);
 472         type = type & ~T_KERNEL;
 473         va = TLB_TAR_VA(data);
 474
 475
 476         if (data > VM_MIN_DIRECT_ADDRESS)
 477                 printf("trap_pfault(type=%ld, data=0x%lx, tpc=0x%lx, ctx=0x%lx)\n",
 478                        type, data, tf->tf_tpc, ctx);
 479
 480 #if 0
 481         CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx",
 482             td, p->p_vmspace->vm_pmap.pm_context, va, ctx);
 483 #endif
 484         KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL"));
 485         KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL"));
 486         KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
 487
 488
 489         if (type == T_DATA_PROTECTION) {
 490                 prot = VM_PROT_WRITE;
 491                 flags = VM_FAULT_DIRTY;
 492         } else {
 493                 if (type == T_DATA_MISS)
 494                         prot = VM_PROT_READ;
 495                 else
 496                         prot = VM_PROT_READ | VM_PROT_EXECUTE;
 497                 flags = VM_FAULT_NORMAL;
 498         }
 499
 500         if (ctx != TLB_CTX_KERNEL) {
 501                 if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
 502                     (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
 503                      tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
 504                         tf->tf_tpc = (u_long)fs_fault;
 505                         tf->tf_tnpc = tf->tf_tpc + 4;
 506                         return (0);
 507                 }
 508
 509                 /*
 510                  * This is a fault on non-kernel virtual memory.
 511                  */
 512                 vm = p->p_vmspace;
 513
 514                 /*
 515                  * Keep swapout from messing with us during this
 516                  * critical time.
 517                  */
 518                 PROC_LOCK(p);
 519                 ++p->p_lock;
 520                 PROC_UNLOCK(p);
 521
 522                 /* Fault in the user page. */
 523                 rv = vm_fault(&vm->vm_map, va, prot, flags);
 524
 525                 /*
 526                  * Now the process can be swapped again.
 527                  */
 528                 PROC_LOCK(p);
 529                 --p->p_lock;
 530                 PROC_UNLOCK(p);
 531         } else {
 532                 /*
 533                  * This is a fault on kernel virtual memory.  Attempts to
 534                  * access kernel memory from user mode cause privileged
 535                  * action traps, not page fault.
 536                  */
 537                 KASSERT(tf->tf_tstate & TSTATE_PRIV,
 538                     ("trap_pfault: fault on nucleus context from user mode"));
 539
 540                 /*
 541                  * Don't have to worry about process locking or stacks in the
 542                  * kernel.
 543                  */
 544                 rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL);
 545         }
 546
 547         CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d",
 548             td, va, rv);
 549         if (rv == KERN_SUCCESS)
 550                 return (0);
 551         if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
 552                 if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
 553                     tf->tf_tpc <= (u_long)fs_nofault_end) {
 554                         tf->tf_tpc = (u_long)fs_fault;
 555                         tf->tf_tnpc = tf->tf_tpc + 4;
 556                         return (0);
 557                 }
 558                 if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
 559                     tf->tf_tpc <= (u_long)copy_nofault_end) {
 560                         tf->tf_tpc = (u_long)copy_fault;
 561                         tf->tf_tnpc = tf->tf_tpc + 4;
 562                         return (0);
 563                 }
 564         }
 565         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 566 }
 567
 568 /* Maximum number of arguments that can be passed via the out registers. */
 569 #define REG_MAXARGS     6
 570
 571 /*
 572  * Syscall handler. The arguments to the syscall are passed in the o registers
 573  * by the caller, and are saved in the trap frame. The syscall number is passed
 574  * in %g1 (and also saved in the trap frame).
 575  */
 576 void
 577 syscall(struct trapframe *tf)
 578 {
 579         struct sysent *callp;
 580         struct thread *td;
 581         register_t args[8];
 582         register_t *argp;
 583         struct proc *p;
 584         u_long code;
 585         u_long tpc;
 586         int reg;
 587         int regcnt;
 588         int narg;
 589         int error;
 590
 591         td = curthread;
 592         KASSERT(td != NULL, ("trap: curthread NULL"));
 593         KASSERT(td->td_proc != NULL, ("trap: curproc NULL"));
 594
 595         p = td->td_proc;
 596
 597         PCPU_INC(cnt.v_syscall);
 598
 599         td->td_pticks = 0;
 600         td->td_frame = tf;
 601         if (td->td_ucred != p->p_ucred)
 602                 cred_update_thread(td);
 603         code = tf->tf_global[1];
 604
 605         /*
 606          * For syscalls, we don't want to retry the faulting instruction
 607          * (usually), instead we need to advance one instruction.
 608          */
 609         tpc = tf->tf_tpc;
 610         TF_DONE(tf);
 611
 612         reg = 0;
 613         regcnt = REG_MAXARGS;
 614         if (p->p_sysent->sv_prepsyscall) {
 615                 /*
 616                  * The prep code is MP aware.
 617                  */
 618 #if 0
 619                 (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params);
 620 #endif
 621         } else  if (code == SYS_syscall || code == SYS___syscall) {
 622                 code = tf->tf_out[reg++];
 623                 regcnt--;
 624         }
 625
 626         if (p->p_sysent->sv_mask)
 627                 code &= p->p_sysent->sv_mask;
 628
 629         if (code >= p->p_sysent->sv_size)
 630                 callp = &p->p_sysent->sv_table[0];
 631         else
 632                 callp = &p->p_sysent->sv_table[code];
 633
 634         narg = callp->sy_narg;
 635
 636         if (narg <= regcnt) {
 637                 argp = &tf->tf_out[reg];
 638                 error = 0;
 639         } else {
 640                 KASSERT(narg <= sizeof(args) / sizeof(args[0]),
 641                     ("Too many syscall arguments!"));
 642                 argp = args;
 643                 bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt);
 644                 error = copyin((void *)(tf->tf_out[6] + SPOFF +
 645                     offsetof(struct frame, fr_pad[6])),
 646                     &args[regcnt], (narg - regcnt) * sizeof(args[0]));
 647         }
 648
 649         CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td,
 650             syscallnames[code], argp[0], argp[1], argp[2]);
 651
 652         /*
 653          * Try to run the syscall without the MP lock if the syscall
 654          * is MP safe.
 655          */
 656 #ifdef KTRACE
 657         if (KTRPOINT(td, KTR_SYSCALL))
 658                 ktrsyscall(code, narg, argp);
 659 #endif
 660         if (error == 0) {
 661                 td->td_retval[0] = 0;
 662                 td->td_retval[1] = 0;
 663
 664                 STOPEVENT(p, S_SCE, narg);      /* MP aware */
 665
 666                 PTRACESTOP_SC(p, td, S_PT_SCE);
 667
 668                 AUDIT_SYSCALL_ENTER(code, td);
 669                 error = (*callp->sy_call)(td, argp);
 670                 AUDIT_SYSCALL_EXIT(error, td);
 671
 672                 CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p,
 673                     error, syscallnames[code], td->td_retval[0],
 674                     td->td_retval[1]);
 675         }
 676
 677         /*
 678          * MP SAFE (we may or may not have the MP lock at this point)
 679          */
 680         switch (error) {
 681         case 0:
 682                 tf->tf_out[0] = td->td_retval[0];
 683                 tf->tf_out[1] = td->td_retval[1];
 684                 tf->tf_tstate &= ~TSTATE_XCC_C;
 685                 break;
 686
 687         case ERESTART:
 688                 /*
 689                  * Undo the tpc advancement we have done above, we want to
 690                  * reexecute the system call.
 691                  */
 692                 tf->tf_tpc = tpc;
 693                 tf->tf_tnpc -= 4;
 694                 break;
 695
 696         case EJUSTRETURN:
 697                 break;
 698
 699         default:
 700                 if (p->p_sysent->sv_errsize) {
 701                         if (error >= p->p_sysent->sv_errsize)
 702                                 error = -1;     /* XXX */
 703                         else
 704                                 error = p->p_sysent->sv_errtbl[error];
 705                 }
 706                 tf->tf_out[0] = error;
 707                 tf->tf_tstate |= TSTATE_XCC_C;
 708                 break;
 709         }
 710
 711         /*
 712          * Handle reschedule and other end-of-syscall issues
 713          */
 714         userret(td, tf);
 715
 716 #ifdef KTRACE
 717         if (KTRPOINT(td, KTR_SYSRET))
 718                 ktrsysret(code, error, td->td_retval[0]);
 719 #endif
 720         /*
 721          * This works because errno is findable through the
 722          * register set.  If we ever support an emulation where this
 723          * is not the case, this code will need to be revisited.
 724          */
 725         STOPEVENT(p, S_SCX, code);
 726
 727         PTRACESTOP_SC(p, td, S_PT_SCX);
 728
 729         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 730             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 731         mtx_assert(&Giant, MA_NOTOWNED);
 732 }