sys/ia64/ia64/trap.c

   1 /*-
   2  * Copyright (c) 2005 Marcel Moolenaar
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_ddb.h"
  31 #include "opt_ktrace.h"
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kdb.h>
  36 #include <sys/ktr.h>
  37 #include <sys/sysproto.h>
  38 #include <sys/kernel.h>
  39 #include <sys/proc.h>
  40 #include <sys/exec.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/sched.h>
  44 #include <sys/smp.h>
  45 #include <sys/vmmeter.h>
  46 #include <sys/sysent.h>
  47 #include <sys/signalvar.h>
  48 #include <sys/syscall.h>
  49 #include <sys/pioctl.h>
  50 #include <sys/ptrace.h>
  51 #include <sys/sysctl.h>
  52 #include <vm/vm.h>
  53 #include <vm/vm_kern.h>
  54 #include <vm/vm_page.h>
  55 #include <vm/vm_map.h>
  56 #include <vm/vm_extern.h>
  57 #include <vm/vm_param.h>
  58 #include <sys/ptrace.h>
  59 #include <machine/cpu.h>
  60 #include <machine/md_var.h>
  61 #include <machine/reg.h>
  62 #include <machine/pal.h>
  63 #include <machine/fpu.h>
  64 #include <machine/efi.h>
  65 #include <machine/pcb.h>
  66 #ifdef SMP
  67 #include <machine/smp.h>
  68 #endif
  69
  70 #ifdef KTRACE
  71 #include <sys/uio.h>
  72 #include <sys/ktrace.h>
  73 #endif
  74
  75 #include <security/audit/audit.h>
  76
  77 #include <ia64/disasm/disasm.h>
  78
  79 static int print_usertrap = 0;
  80 SYSCTL_INT(_machdep, OID_AUTO, print_usertrap,
  81     CTLFLAG_RW, &print_usertrap, 0, "");
  82
  83 static void break_syscall(struct trapframe *tf);
  84
  85 /*
  86  * EFI-Provided FPSWA interface (Floating Point SoftWare Assist)
  87  */
  88 extern struct fpswa_iface *fpswa_iface;
  89
  90 extern char *syscallnames[];
  91
  92 static const char *ia64_vector_names[] = {
  93         "VHPT Translation",                     /* 0 */
  94         "Instruction TLB",                      /* 1 */
  95         "Data TLB",                             /* 2 */
  96         "Alternate Instruction TLB",            /* 3 */
  97         "Alternate Data TLB",                   /* 4 */
  98         "Data Nested TLB",                      /* 5 */
  99         "Instruction Key Miss",                 /* 6 */
 100         "Data Key Miss",                        /* 7 */
 101         "Dirty-Bit",                            /* 8 */
 102         "Instruction Access-Bit",               /* 9 */
 103         "Data Access-Bit",                      /* 10 */
 104         "Break Instruction",                    /* 11 */
 105         "External Interrupt",                   /* 12 */
 106         "Reserved 13",                          /* 13 */
 107         "Reserved 14",                          /* 14 */
 108         "Reserved 15",                          /* 15 */
 109         "Reserved 16",                          /* 16 */
 110         "Reserved 17",                          /* 17 */
 111         "Reserved 18",                          /* 18 */
 112         "Reserved 19",                          /* 19 */
 113         "Page Not Present",                     /* 20 */
 114         "Key Permission",                       /* 21 */
 115         "Instruction Access Rights",            /* 22 */
 116         "Data Access Rights",                   /* 23 */
 117         "General Exception",                    /* 24 */
 118         "Disabled FP-Register",                 /* 25 */
 119         "NaT Consumption",                      /* 26 */
 120         "Speculation",                          /* 27 */
 121         "Reserved 28",                          /* 28 */
 122         "Debug",                                /* 29 */
 123         "Unaligned Reference",                  /* 30 */
 124         "Unsupported Data Reference",           /* 31 */
 125         "Floating-point Fault",                 /* 32 */
 126         "Floating-point Trap",                  /* 33 */
 127         "Lower-Privilege Transfer Trap",        /* 34 */
 128         "Taken Branch Trap",                    /* 35 */
 129         "Single Step Trap",                     /* 36 */
 130         "Reserved 37",                          /* 37 */
 131         "Reserved 38",                          /* 38 */
 132         "Reserved 39",                          /* 39 */
 133         "Reserved 40",                          /* 40 */
 134         "Reserved 41",                          /* 41 */
 135         "Reserved 42",                          /* 42 */
 136         "Reserved 43",                          /* 43 */
 137         "Reserved 44",                          /* 44 */
 138         "IA-32 Exception",                      /* 45 */
 139         "IA-32 Intercept",                      /* 46 */
 140         "IA-32 Interrupt",                      /* 47 */
 141         "Reserved 48",                          /* 48 */
 142         "Reserved 49",                          /* 49 */
 143         "Reserved 50",                          /* 50 */
 144         "Reserved 51",                          /* 51 */
 145         "Reserved 52",                          /* 52 */
 146         "Reserved 53",                          /* 53 */
 147         "Reserved 54",                          /* 54 */
 148         "Reserved 55",                          /* 55 */
 149         "Reserved 56",                          /* 56 */
 150         "Reserved 57",                          /* 57 */
 151         "Reserved 58",                          /* 58 */
 152         "Reserved 59",                          /* 59 */
 153         "Reserved 60",                          /* 60 */
 154         "Reserved 61",                          /* 61 */
 155         "Reserved 62",                          /* 62 */
 156         "Reserved 63",                          /* 63 */
 157         "Reserved 64",                          /* 64 */
 158         "Reserved 65",                          /* 65 */
 159         "Reserved 66",                          /* 66 */
 160         "Reserved 67",                          /* 67 */
 161 };
 162
 163 struct bitname {
 164         uint64_t mask;
 165         const char* name;
 166 };
 167
 168 static void
 169 printbits(uint64_t mask, struct bitname *bn, int count)
 170 {
 171         int i, first = 1;
 172         uint64_t bit;
 173
 174         for (i = 0; i < count; i++) {
 175                 /*
 176                  * Handle fields wider than one bit.
 177                  */
 178                 bit = bn[i].mask & ~(bn[i].mask - 1);
 179                 if (bn[i].mask > bit) {
 180                         if (first)
 181                                 first = 0;
 182                         else
 183                                 printf(",");
 184                         printf("%s=%ld", bn[i].name,
 185                                (mask & bn[i].mask) / bit);
 186                 } else if (mask & bit) {
 187                         if (first)
 188                                 first = 0;
 189                         else
 190                                 printf(",");
 191                         printf("%s", bn[i].name);
 192                 }
 193         }
 194 }
 195
 196 struct bitname psr_bits[] = {
 197         {IA64_PSR_BE,   "be"},
 198         {IA64_PSR_UP,   "up"},
 199         {IA64_PSR_AC,   "ac"},
 200         {IA64_PSR_MFL,  "mfl"},
 201         {IA64_PSR_MFH,  "mfh"},
 202         {IA64_PSR_IC,   "ic"},
 203         {IA64_PSR_I,    "i"},
 204         {IA64_PSR_PK,   "pk"},
 205         {IA64_PSR_DT,   "dt"},
 206         {IA64_PSR_DFL,  "dfl"},
 207         {IA64_PSR_DFH,  "dfh"},
 208         {IA64_PSR_SP,   "sp"},
 209         {IA64_PSR_PP,   "pp"},
 210         {IA64_PSR_DI,   "di"},
 211         {IA64_PSR_SI,   "si"},
 212         {IA64_PSR_DB,   "db"},
 213         {IA64_PSR_LP,   "lp"},
 214         {IA64_PSR_TB,   "tb"},
 215         {IA64_PSR_RT,   "rt"},
 216         {IA64_PSR_CPL,  "cpl"},
 217         {IA64_PSR_IS,   "is"},
 218         {IA64_PSR_MC,   "mc"},
 219         {IA64_PSR_IT,   "it"},
 220         {IA64_PSR_ID,   "id"},
 221         {IA64_PSR_DA,   "da"},
 222         {IA64_PSR_DD,   "dd"},
 223         {IA64_PSR_SS,   "ss"},
 224         {IA64_PSR_RI,   "ri"},
 225         {IA64_PSR_ED,   "ed"},
 226         {IA64_PSR_BN,   "bn"},
 227         {IA64_PSR_IA,   "ia"},
 228 };
 229
 230 static void
 231 printpsr(uint64_t psr)
 232 {
 233         printbits(psr, psr_bits, sizeof(psr_bits)/sizeof(psr_bits[0]));
 234 }
 235
 236 struct bitname isr_bits[] = {
 237         {IA64_ISR_CODE, "code"},
 238         {IA64_ISR_VECTOR, "vector"},
 239         {IA64_ISR_X,    "x"},
 240         {IA64_ISR_W,    "w"},
 241         {IA64_ISR_R,    "r"},
 242         {IA64_ISR_NA,   "na"},
 243         {IA64_ISR_SP,   "sp"},
 244         {IA64_ISR_RS,   "rs"},
 245         {IA64_ISR_IR,   "ir"},
 246         {IA64_ISR_NI,   "ni"},
 247         {IA64_ISR_SO,   "so"},
 248         {IA64_ISR_EI,   "ei"},
 249         {IA64_ISR_ED,   "ed"},
 250 };
 251
 252 static void printisr(uint64_t isr)
 253 {
 254         printbits(isr, isr_bits, sizeof(isr_bits)/sizeof(isr_bits[0]));
 255 }
 256
 257 static void
 258 printtrap(int vector, struct trapframe *tf, int isfatal, int user)
 259 {
 260         printf("\n");
 261         printf("%s %s trap (cpu %d):\n", isfatal? "fatal" : "handled",
 262                user ? "user" : "kernel", PCPU_GET(cpuid));
 263         printf("\n");
 264         printf("    trap vector = 0x%x (%s)\n",
 265                vector, ia64_vector_names[vector]);
 266         printf("    cr.iip      = 0x%lx\n", tf->tf_special.iip);
 267         printf("    cr.ipsr     = 0x%lx (", tf->tf_special.psr);
 268         printpsr(tf->tf_special.psr);
 269         printf(")\n");
 270         printf("    cr.isr      = 0x%lx (", tf->tf_special.isr);
 271         printisr(tf->tf_special.isr);
 272         printf(")\n");
 273         printf("    cr.ifa      = 0x%lx\n", tf->tf_special.ifa);
 274         if (tf->tf_special.psr & IA64_PSR_IS) {
 275                 printf("    ar.cflg     = 0x%lx\n", ia64_get_cflg());
 276                 printf("    ar.csd      = 0x%lx\n", ia64_get_csd());
 277                 printf("    ar.ssd      = 0x%lx\n", ia64_get_ssd());
 278         }
 279         printf("    curthread   = %p\n", curthread);
 280         if (curthread != NULL)
 281                 printf("        pid = %d, comm = %s\n",
 282                        curthread->td_proc->p_pid, curthread->td_proc->p_comm);
 283         printf("\n");
 284 }
 285
 286 /*
 287  * We got a trap caused by a break instruction and the immediate was 0.
 288  * This indicates that we may have a break.b with some non-zero immediate.
 289  * The break.b doesn't cause the immediate to be put in cr.iim.  Hence,
 290  * we need to disassemble the bundle and return the immediate found there.
 291  * This may be a 0 value anyway.  Return 0 for any error condition.  This
 292  * will result in a SIGILL, which is pretty much the best thing to do.
 293  */
 294 static uint64_t
 295 trap_decode_break(struct trapframe *tf)
 296 {
 297         struct asm_bundle bundle;
 298         struct asm_inst *inst;
 299         int slot;
 300
 301         if (!asm_decode(tf->tf_special.iip, &bundle))
 302                 return (0);
 303
 304         slot = ((tf->tf_special.psr & IA64_PSR_RI) == IA64_PSR_RI_0) ? 0 :
 305             ((tf->tf_special.psr & IA64_PSR_RI) == IA64_PSR_RI_1) ? 1 : 2;
 306         inst = bundle.b_inst + slot;
 307
 308         /*
 309          * Sanity checking: It must be a break instruction and the operand
 310          * that has the break value must be an immediate.
 311          */
 312         if (inst->i_op != ASM_OP_BREAK ||
 313             inst->i_oper[1].o_type != ASM_OPER_IMM)
 314                 return (0);
 315
 316         return (inst->i_oper[1].o_value);
 317 }
 318
 319 void
 320 trap_panic(int vector, struct trapframe *tf)
 321 {
 322
 323         printtrap(vector, tf, 1, TRAPF_USERMODE(tf));
 324 #ifdef KDB
 325         kdb_trap(vector, 0, tf);
 326 #endif
 327         panic("trap");
 328 }
 329
 330 /*
 331  *
 332  */
 333 int
 334 do_ast(struct trapframe *tf)
 335 {
 336
 337         disable_intr();
 338         while (curthread->td_flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)) {
 339                 enable_intr();
 340                 ast(tf);
 341                 disable_intr();
 342         }
 343         /*
 344          * Keep interrupts disabled. We return r10 as a favor to the EPC
 345          * syscall code so that it can quicky determine if the syscall
 346          * needs to be restarted or not.
 347          */
 348         return (tf->tf_scratch.gr10);
 349 }
 350
 351 /*
 352  * Trap is called from exception.s to handle most types of processor traps.
 353  */
 354 /*ARGSUSED*/
 355 void
 356 trap(int vector, struct trapframe *tf)
 357 {
 358         struct proc *p;
 359         struct thread *td;
 360         uint64_t ucode;
 361         int error, sig, user;
 362         ksiginfo_t ksi;
 363
 364         user = TRAPF_USERMODE(tf) ? 1 : 0;
 365
 366         PCPU_INC(cnt.v_trap);
 367
 368         td = curthread;
 369         p = td->td_proc;
 370         ucode = 0;
 371
 372         if (user) {
 373                 ia64_set_fpsr(IA64_FPSR_DEFAULT);
 374                 td->td_pticks = 0;
 375                 td->td_frame = tf;
 376                 if (td->td_ucred != p->p_ucred)
 377                         cred_update_thread(td);
 378         } else {
 379                 KASSERT(cold || td->td_ucred != NULL,
 380                     ("kernel trap doesn't have ucred"));
 381 #ifdef KDB
 382                 if (kdb_active)
 383                         kdb_reenter();
 384 #endif
 385         }
 386
 387         sig = 0;
 388         switch (vector) {
 389         case IA64_VEC_VHPT:
 390                 /*
 391                  * This one is tricky. We should hardwire the VHPT, but
 392                  * don't at this time. I think we're mostly lucky that
 393                  * the VHPT is mapped.
 394                  */
 395                 trap_panic(vector, tf);
 396                 break;
 397
 398         case IA64_VEC_ITLB:
 399         case IA64_VEC_DTLB:
 400         case IA64_VEC_EXT_INTR:
 401                 /* We never call trap() with these vectors. */
 402                 trap_panic(vector, tf);
 403                 break;
 404
 405         case IA64_VEC_ALT_ITLB:
 406         case IA64_VEC_ALT_DTLB:
 407                 /*
 408                  * These should never happen, because regions 0-4 use the
 409                  * VHPT. If we get one of these it means we didn't program
 410                  * the region registers correctly.
 411                  */
 412                 trap_panic(vector, tf);
 413                 break;
 414
 415         case IA64_VEC_NESTED_DTLB:
 416                 /*
 417                  * We never call trap() with this vector. We may want to
 418                  * do that in the future in case the nested TLB handler
 419                  * could not find the translation it needs. In that case
 420                  * we could switch to a special (hardwired) stack and
 421                  * come here to produce a nice panic().
 422                  */
 423                 trap_panic(vector, tf);
 424                 break;
 425
 426         case IA64_VEC_IKEY_MISS:
 427         case IA64_VEC_DKEY_MISS:
 428         case IA64_VEC_KEY_PERMISSION:
 429                 /*
 430                  * We don't use protection keys, so we should never get
 431                  * these faults.
 432                  */
 433                 trap_panic(vector, tf);
 434                 break;
 435
 436         case IA64_VEC_DIRTY_BIT:
 437         case IA64_VEC_INST_ACCESS:
 438         case IA64_VEC_DATA_ACCESS:
 439                 /*
 440                  * We get here if we read or write to a page of which the
 441                  * PTE does not have the access bit or dirty bit set and
 442                  * we can not find the PTE in our datastructures. This
 443                  * either means we have a stale PTE in the TLB, or we lost
 444                  * the PTE in our datastructures.
 445                  */
 446                 trap_panic(vector, tf);
 447                 break;
 448
 449         case IA64_VEC_BREAK:
 450                 if (user) {
 451                         ucode = (int)tf->tf_special.ifa & 0x1FFFFF;
 452                         if (ucode == 0) {
 453                                 /*
 454                                  * A break.b doesn't cause the immediate to be
 455                                  * stored in cr.iim (and saved in the TF in
 456                                  * tf_special.ifa).  We need to decode the
 457                                  * instruction to find out what the immediate
 458                                  * was.  Note that if the break instruction
 459                                  * didn't happen to be a break.b, but any
 460                                  * other break with an immediate of 0, we
 461                                  * will do unnecessary work to get the value
 462                                  * we already had.  Not an issue, because a
 463                                  * break 0 is invalid.
 464                                  */
 465                                 ucode = trap_decode_break(tf);
 466                         }
 467                         if (ucode < 0x80000) {
 468                                 /* Software interrupts. */
 469                                 switch (ucode) {
 470                                 case 0:         /* Unknown error. */
 471                                         sig = SIGILL;
 472                                         break;
 473                                 case 1:         /* Integer divide by zero. */
 474                                         sig = SIGFPE;
 475                                         ucode = FPE_INTDIV;
 476                                         break;
 477                                 case 2:         /* Integer overflow. */
 478                                         sig = SIGFPE;
 479                                         ucode = FPE_INTOVF;
 480                                         break;
 481                                 case 3:         /* Range check/bounds check. */
 482                                         sig = SIGFPE;
 483                                         ucode = FPE_FLTSUB;
 484                                         break;
 485                                 case 6:         /* Decimal overflow. */
 486                                 case 7:         /* Decimal divide by zero. */
 487                                 case 8:         /* Packed decimal error. */
 488                                 case 9:         /* Invalid ASCII digit. */
 489                                 case 10:        /* Invalid decimal digit. */
 490                                         sig = SIGFPE;
 491                                         ucode = FPE_FLTINV;
 492                                         break;
 493                                 case 4:         /* Null pointer dereference. */
 494                                 case 5:         /* Misaligned data. */
 495                                 case 11:        /* Paragraph stack overflow. */
 496                                         sig = SIGSEGV;
 497                                         break;
 498                                 default:
 499                                         sig = SIGILL;
 500                                         break;
 501                                 }
 502                         } else if (ucode < 0x100000) {
 503                                 /* Debugger breakpoint. */
 504                                 tf->tf_special.psr &= ~IA64_PSR_SS;
 505                                 sig = SIGTRAP;
 506                         } else if (ucode == 0x100000) {
 507                                 break_syscall(tf);
 508                                 return;         /* do_ast() already called. */
 509                         } else if (ucode == 0x180000) {
 510                                 mcontext_t mc;
 511
 512                                 error = copyin((void*)tf->tf_scratch.gr8,
 513                                     &mc, sizeof(mc));
 514                                 if (!error) {
 515                                         set_mcontext(td, &mc);
 516                                         return; /* Don't call do_ast()!!! */
 517                                 }
 518                                 sig = SIGSEGV;
 519                                 ucode = tf->tf_scratch.gr8;
 520                         } else
 521                                 sig = SIGILL;
 522                 } else {
 523 #ifdef KDB
 524                         if (kdb_trap(vector, 0, tf))
 525                                 return;
 526                         panic("trap");
 527 #else
 528                         trap_panic(vector, tf);
 529 #endif
 530                 }
 531                 break;
 532
 533         case IA64_VEC_PAGE_NOT_PRESENT:
 534         case IA64_VEC_INST_ACCESS_RIGHTS:
 535         case IA64_VEC_DATA_ACCESS_RIGHTS: {
 536                 vm_offset_t va;
 537                 struct vmspace *vm;
 538                 vm_map_t map;
 539                 vm_prot_t ftype;
 540                 int rv;
 541
 542                 rv = 0;
 543                 va = trunc_page(tf->tf_special.ifa);
 544
 545                 if (va >= VM_MAX_ADDRESS) {
 546                         /*
 547                          * Don't allow user-mode faults for kernel virtual
 548                          * addresses, including the gateway page.
 549                          */
 550                         if (user)
 551                                 goto no_fault_in;
 552                         map = kernel_map;
 553                 } else {
 554                         vm = (p != NULL) ? p->p_vmspace : NULL;
 555                         if (vm == NULL)
 556                                 goto no_fault_in;
 557                         map = &vm->vm_map;
 558                 }
 559
 560                 if (tf->tf_special.isr & IA64_ISR_X)
 561                         ftype = VM_PROT_EXECUTE;
 562                 else if (tf->tf_special.isr & IA64_ISR_W)
 563                         ftype = VM_PROT_WRITE;
 564                 else
 565                         ftype = VM_PROT_READ;
 566
 567                 if (map != kernel_map) {
 568                         /*
 569                          * Keep swapout from messing with us during this
 570                          * critical time.
 571                          */
 572                         PROC_LOCK(p);
 573                         ++p->p_lock;
 574                         PROC_UNLOCK(p);
 575
 576                         /* Fault in the user page: */
 577                         rv = vm_fault(map, va, ftype, (ftype & VM_PROT_WRITE)
 578                             ? VM_FAULT_DIRTY : VM_FAULT_NORMAL);
 579
 580                         PROC_LOCK(p);
 581                         --p->p_lock;
 582                         PROC_UNLOCK(p);
 583                 } else {
 584                         /*
 585                          * Don't have to worry about process locking or
 586                          * stacks in the kernel.
 587                          */
 588                         rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
 589                 }
 590
 591                 if (rv == KERN_SUCCESS)
 592                         goto out;
 593
 594         no_fault_in:
 595                 if (!user) {
 596                         /* Check for copyin/copyout fault. */
 597                         if (td != NULL && td->td_pcb->pcb_onfault != 0) {
 598                                 tf->tf_special.iip =
 599                                     td->td_pcb->pcb_onfault;
 600                                 tf->tf_special.psr &= ~IA64_PSR_RI;
 601                                 td->td_pcb->pcb_onfault = 0;
 602                                 goto out;
 603                         }
 604                         trap_panic(vector, tf);
 605                 }
 606                 ucode = va;
 607                 sig = (rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV;
 608                 break;
 609         }
 610
 611         case IA64_VEC_GENERAL_EXCEPTION: {
 612                 int code;
 613
 614                 if (!user)
 615                         trap_panic(vector, tf);
 616
 617                 code = tf->tf_special.isr & (IA64_ISR_CODE & 0xf0ull);
 618                 switch (code) {
 619                 case 0x0:       /* Illegal Operation Fault. */
 620                         sig = ia64_emulate(tf, td);
 621                         break;
 622                 default:
 623                         sig = SIGILL;
 624                         break;
 625                 }
 626                 if (sig == 0)
 627                         goto out;
 628                 ucode = vector;
 629                 break;
 630         }
 631
 632         case IA64_VEC_SPECULATION:
 633                 /*
 634                  * The branching behaviour of the chk instruction is not
 635                  * implemented by the processor. All we need to do is
 636                  * compute the target address of the branch and make sure
 637                  * that control is transfered to that address.
 638                  * We should do this in the IVT table and not by entring
 639                  * the kernel...
 640                  */
 641                 tf->tf_special.iip += tf->tf_special.ifa << 4;
 642                 tf->tf_special.psr &= ~IA64_PSR_RI;
 643                 goto out;
 644
 645         case IA64_VEC_NAT_CONSUMPTION:
 646         case IA64_VEC_UNSUPP_DATA_REFERENCE:
 647                 if (user) {
 648                         ucode = vector;
 649                         sig = SIGILL;
 650                 } else
 651                         trap_panic(vector, tf);
 652                 break;
 653
 654         case IA64_VEC_DISABLED_FP: {
 655                 struct pcpu *pcpu;
 656                 struct pcb *pcb;
 657                 struct thread *thr;
 658
 659                 /* Always fatal in kernel. Should never happen. */
 660                 if (!user)
 661                         trap_panic(vector, tf);
 662
 663                 sched_pin();
 664                 thr = PCPU_GET(fpcurthread);
 665                 if (thr == td) {
 666                         /*
 667                          * Short-circuit handling the trap when this CPU
 668                          * already holds the high FP registers for this
 669                          * thread.  We really shouldn't get the trap in the
 670                          * first place, but since it's only a performance
 671                          * issue and not a correctness issue, we emit a
 672                          * message for now, enable the high FP registers and
 673                          * return.
 674                          */
 675                         printf("XXX: bogusly disabled high FP regs\n");
 676                         tf->tf_special.psr &= ~IA64_PSR_DFH;
 677                         sched_unpin();
 678                         goto out;
 679                 } else if (thr != NULL) {
 680                         mtx_lock_spin(&thr->td_md.md_highfp_mtx);
 681                         pcb = thr->td_pcb;
 682                         save_high_fp(&pcb->pcb_high_fp);
 683                         pcb->pcb_fpcpu = NULL;
 684                         PCPU_SET(fpcurthread, NULL);
 685                         mtx_unlock_spin(&thr->td_md.md_highfp_mtx);
 686                         thr = NULL;
 687                 }
 688
 689                 mtx_lock_spin(&td->td_md.md_highfp_mtx);
 690                 pcb = td->td_pcb;
 691                 pcpu = pcb->pcb_fpcpu;
 692
 693 #ifdef SMP
 694                 if (pcpu != NULL) {
 695                         mtx_unlock_spin(&td->td_md.md_highfp_mtx);
 696                         ipi_send(pcpu, IPI_HIGH_FP);
 697                         while (pcb->pcb_fpcpu == pcpu)
 698                                 DELAY(100);
 699                         mtx_lock_spin(&td->td_md.md_highfp_mtx);
 700                         pcpu = pcb->pcb_fpcpu;
 701                         thr = PCPU_GET(fpcurthread);
 702                 }
 703 #endif
 704
 705                 if (thr == NULL && pcpu == NULL) {
 706                         restore_high_fp(&pcb->pcb_high_fp);
 707                         PCPU_SET(fpcurthread, td);
 708                         pcb->pcb_fpcpu = pcpup;
 709                         tf->tf_special.psr &= ~IA64_PSR_MFH;
 710                         tf->tf_special.psr &= ~IA64_PSR_DFH;
 711                 }
 712
 713                 mtx_unlock_spin(&td->td_md.md_highfp_mtx);
 714                 sched_unpin();
 715                 goto out;
 716         }
 717
 718         case IA64_VEC_DEBUG:
 719         case IA64_VEC_SINGLE_STEP_TRAP:
 720                 tf->tf_special.psr &= ~IA64_PSR_SS;
 721                 if (!user) {
 722 #ifdef KDB
 723                         if (kdb_trap(vector, 0, tf))
 724                                 return;
 725                         panic("trap");
 726 #else
 727                         trap_panic(vector, tf);
 728 #endif
 729                 }
 730                 sig = SIGTRAP;
 731                 break;
 732
 733         case IA64_VEC_UNALIGNED_REFERENCE:
 734                 /*
 735                  * If user-land, do whatever fixups, printing, and
 736                  * signalling is appropriate (based on system-wide
 737                  * and per-process unaligned-access-handling flags).
 738                  */
 739                 if (user) {
 740                         sig = unaligned_fixup(tf, td);
 741                         if (sig == 0)
 742                                 goto out;
 743                         ucode = tf->tf_special.ifa;     /* VA */
 744                 } else {
 745                         /* Check for copyin/copyout fault. */
 746                         if (td != NULL && td->td_pcb->pcb_onfault != 0) {
 747                                 tf->tf_special.iip =
 748                                     td->td_pcb->pcb_onfault;
 749                                 tf->tf_special.psr &= ~IA64_PSR_RI;
 750                                 td->td_pcb->pcb_onfault = 0;
 751                                 goto out;
 752                         }
 753                         trap_panic(vector, tf);
 754                 }
 755                 break;
 756
 757         case IA64_VEC_FLOATING_POINT_FAULT:
 758         case IA64_VEC_FLOATING_POINT_TRAP: {
 759                 struct fpswa_bundle bundle;
 760                 struct fpswa_fpctx fpctx;
 761                 struct fpswa_ret ret;
 762                 char *ip;
 763                 u_long fault;
 764
 765                 /* Always fatal in kernel. Should never happen. */
 766                 if (!user)
 767                         trap_panic(vector, tf);
 768
 769                 if (fpswa_iface == NULL) {
 770                         sig = SIGFPE;
 771                         ucode = 0;
 772                         break;
 773                 }
 774
 775                 ip = (char *)tf->tf_special.iip;
 776                 if (vector == IA64_VEC_FLOATING_POINT_TRAP &&
 777                     (tf->tf_special.psr & IA64_PSR_RI) == 0)
 778                         ip -= 16;
 779                 error = copyin(ip, &bundle, sizeof(bundle));
 780                 if (error) {
 781                         sig = SIGBUS;   /* EFAULT, basically */
 782                         ucode = 0;      /* exception summary */
 783                         break;
 784                 }
 785
 786                 /* f6-f15 are saved in exception_save */
 787                 fpctx.mask_low = 0xffc0;                /* bits 6 - 15 */
 788                 fpctx.mask_high = 0;
 789                 fpctx.fp_low_preserved = NULL;
 790                 fpctx.fp_low_volatile = &tf->tf_scratch_fp.fr6;
 791                 fpctx.fp_high_preserved = NULL;
 792                 fpctx.fp_high_volatile = NULL;
 793
 794                 fault = (vector == IA64_VEC_FLOATING_POINT_FAULT) ? 1 : 0;
 795
 796                 /*
 797                  * We have the high FP registers disabled while in the
 798                  * kernel. Enable them for the FPSWA handler only.
 799                  */
 800                 ia64_enable_highfp();
 801
 802                 /* The docs are unclear.  Is Fpswa reentrant? */
 803                 ret = fpswa_iface->if_fpswa(fault, &bundle,
 804                     &tf->tf_special.psr, &tf->tf_special.fpsr,
 805                     &tf->tf_special.isr, &tf->tf_special.pr,
 806                     &tf->tf_special.cfm, &fpctx);
 807
 808                 ia64_disable_highfp();
 809
 810                 /*
 811                  * Update ipsr and iip to next instruction. We only
 812                  * have to do that for faults.
 813                  */
 814                 if (fault && (ret.status == 0 || (ret.status & 2))) {
 815                         int ei;
 816
 817                         ei = (tf->tf_special.isr >> 41) & 0x03;
 818                         if (ei == 0) {          /* no template for this case */
 819                                 tf->tf_special.psr &= ~IA64_ISR_EI;
 820                                 tf->tf_special.psr |= IA64_ISR_EI_1;
 821                         } else if (ei == 1) {   /* MFI or MFB */
 822                                 tf->tf_special.psr &= ~IA64_ISR_EI;
 823                                 tf->tf_special.psr |= IA64_ISR_EI_2;
 824                         } else if (ei == 2) {   /* MMF */
 825                                 tf->tf_special.psr &= ~IA64_ISR_EI;
 826                                 tf->tf_special.iip += 0x10;
 827                         }
 828                 }
 829
 830                 if (ret.status == 0) {
 831                         goto out;
 832                 } else if (ret.status == -1) {
 833                         printf("FATAL: FPSWA err1 %lx, err2 %lx, err3 %lx\n",
 834                             ret.err1, ret.err2, ret.err3);
 835                         panic("fpswa fatal error on fp fault");
 836                 } else {
 837                         sig = SIGFPE;
 838                         ucode = 0;              /* XXX exception summary */
 839                         break;
 840                 }
 841         }
 842
 843         case IA64_VEC_LOWER_PRIVILEGE_TRANSFER:
 844                 /*
 845                  * The lower-privilege transfer trap is used by the EPC
 846                  * syscall code to trigger re-entry into the kernel when the
 847                  * process should be single stepped. The problem is that
 848                  * there's no way to set single stepping directly without
 849                  * using the rfi instruction. So instead we enable the
 850                  * lower-privilege transfer trap and when we get here we
 851                  * know that the process is about to enter userland (and
 852                  * has already lowered its privilege).
 853                  * However, there's another gotcha. When the process has
 854                  * lowered it's privilege it's still running in the gateway
 855                  * page. If we enable single stepping, we'll be stepping
 856                  * the code in the gateway page. In and by itself this is
 857                  * not a problem, but it's an address debuggers won't know
 858                  * anything about. Hence, it can only cause confusion.
 859                  * We know that we need to branch to get out of the gateway
 860                  * page, so what we do here is enable the taken branch
 861                  * trap and just let the process continue. When we branch
 862                  * out of the gateway page we'll get back into the kernel
 863                  * and then we enable single stepping.
 864                  * Since this a rather round-about way of enabling single
 865                  * stepping, don't make things complicated even more by
 866                  * calling userret() and do_ast(). We do that later...
 867                  */
 868                 tf->tf_special.psr &= ~IA64_PSR_LP;
 869                 tf->tf_special.psr |= IA64_PSR_TB;
 870                 return;
 871
 872         case IA64_VEC_TAKEN_BRANCH_TRAP:
 873                 /*
 874                  * Don't assume there aren't any branches other than the
 875                  * branch that takes us out of the gateway page. Check the
 876                  * iip and raise SIGTRAP only when it's an user address.
 877                  */
 878                 if (tf->tf_special.iip >= VM_MAX_ADDRESS)
 879                         return;
 880                 tf->tf_special.psr &= ~IA64_PSR_TB;
 881                 sig = SIGTRAP;
 882                 break;
 883
 884         case IA64_VEC_IA32_EXCEPTION:
 885         case IA64_VEC_IA32_INTERCEPT:
 886         case IA64_VEC_IA32_INTERRUPT:
 887                 sig = SIGEMT;
 888                 ucode = tf->tf_special.iip;
 889                 break;
 890
 891         default:
 892                 /* Reserved vectors get here. Should never happen of course. */
 893                 trap_panic(vector, tf);
 894                 break;
 895         }
 896
 897         KASSERT(sig != 0, ("foo"));
 898
 899         if (print_usertrap)
 900                 printtrap(vector, tf, 1, user);
 901
 902         ksiginfo_init(&ksi);
 903         ksi.ksi_signo = sig;
 904         ksi.ksi_code = ucode;
 905         trapsignal(td, &ksi);
 906
 907 out:
 908         if (user) {
 909                 userret(td, tf);
 910                 mtx_assert(&Giant, MA_NOTOWNED);
 911                 do_ast(tf);
 912         }
 913         return;
 914 }
 915
 916 /*
 917  * Handle break instruction based system calls.
 918  */
 919 void
 920 break_syscall(struct trapframe *tf)
 921 {
 922         uint64_t *bsp, *tfp;
 923         uint64_t iip, psr;
 924         int error, nargs;
 925
 926         /* Save address of break instruction. */
 927         iip = tf->tf_special.iip;
 928         psr = tf->tf_special.psr;
 929
 930         /* Advance to the next instruction. */
 931         tf->tf_special.psr += IA64_PSR_RI_1;
 932         if ((tf->tf_special.psr & IA64_PSR_RI) > IA64_PSR_RI_2) {
 933                 tf->tf_special.iip += 16;
 934                 tf->tf_special.psr &= ~IA64_PSR_RI;
 935         }
 936
 937         /*
 938          * Copy the arguments on the register stack into the trapframe
 939          * to avoid having interleaved NaT collections.
 940          */
 941         tfp = &tf->tf_scratch.gr16;
 942         nargs = tf->tf_special.cfm & 0x7f;
 943         bsp = (uint64_t*)(curthread->td_kstack + tf->tf_special.ndirty +
 944             (tf->tf_special.bspstore & 0x1ffUL));
 945         bsp -= (((uintptr_t)bsp & 0x1ff) < (nargs << 3)) ? (nargs + 1): nargs;
 946         while (nargs--) {
 947                 *tfp++ = *bsp++;
 948                 if (((uintptr_t)bsp & 0x1ff) == 0x1f8)
 949                         bsp++;
 950         }
 951         error = syscall(tf);
 952         if (error == ERESTART) {
 953                 tf->tf_special.iip = iip;
 954                 tf->tf_special.psr = psr;
 955         }
 956
 957         do_ast(tf);
 958 }
 959
 960 /*
 961  * Process a system call.
 962  *
 963  * See syscall.s for details as to how we get here. In order to support
 964  * the ERESTART case, we return the error to our caller. They deal with
 965  * the hairy details.
 966  */
 967 int
 968 syscall(struct trapframe *tf)
 969 {
 970         struct sysent *callp;
 971         struct proc *p;
 972         struct thread *td;
 973         uint64_t *args;
 974         int code, error;
 975
 976         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 977
 978         code = tf->tf_scratch.gr15;
 979         args = &tf->tf_scratch.gr16;
 980
 981         PCPU_INC(cnt.v_syscall);
 982
 983         td = curthread;
 984         td->td_frame = tf;
 985         p = td->td_proc;
 986
 987         td->td_pticks = 0;
 988         if (td->td_ucred != p->p_ucred)
 989                 cred_update_thread(td);
 990 #ifdef KSE
 991         if (p->p_flag & P_SA)
 992                 thread_user_enter(td);
 993 #endif
 994
 995         if (p->p_sysent->sv_prepsyscall) {
 996                 /* (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params); */
 997                 panic("prepsyscall");
 998         } else {
 999                 /*
1000                  * syscall() and __syscall() are handled the same on
1001                  * the ia64, as everything is 64-bit aligned, anyway.
1002                  */
1003                 if (code == SYS_syscall || code == SYS___syscall) {
1004                         /*
1005                          * Code is first argument, followed by actual args.
1006                          */
1007                         code = args[0];
1008                         args++;
1009                 }
1010         }
1011
1012         if (p->p_sysent->sv_mask)
1013                 code &= p->p_sysent->sv_mask;
1014
1015         if (code >= p->p_sysent->sv_size)
1016                 callp = &p->p_sysent->sv_table[0];
1017         else
1018                 callp = &p->p_sysent->sv_table[code];
1019
1020 #ifdef KTRACE
1021         if (KTRPOINT(td, KTR_SYSCALL))
1022                 ktrsyscall(code, callp->sy_narg, args);
1023 #endif
1024         CTR4(KTR_SYSC, "syscall enter thread %p pid %d proc %s code %d", td,
1025             td->td_proc->p_pid, td->td_proc->p_comm, code);
1026
1027         td->td_retval[0] = 0;
1028         td->td_retval[1] = 0;
1029         tf->tf_scratch.gr10 = EJUSTRETURN;
1030
1031         STOPEVENT(p, S_SCE, callp->sy_narg);
1032
1033         PTRACESTOP_SC(p, td, S_PT_SCE);
1034
1035         AUDIT_SYSCALL_ENTER(code, td);
1036         error = (*callp->sy_call)(td, args);
1037         AUDIT_SYSCALL_EXIT(error, td);
1038
1039         if (error != EJUSTRETURN) {
1040                 /*
1041                  * Save the "raw" error code in r10. We use this to handle
1042                  * syscall restarts (see do_ast()).
1043                  */
1044                 tf->tf_scratch.gr10 = error;
1045                 if (error == 0) {
1046                         tf->tf_scratch.gr8 = td->td_retval[0];
1047                         tf->tf_scratch.gr9 = td->td_retval[1];
1048                 } else if (error != ERESTART) {
1049                         if (error < p->p_sysent->sv_errsize)
1050                                 error = p->p_sysent->sv_errtbl[error];
1051                         /*
1052                          * Translated error codes are returned in r8. User
1053                          * processes use the translated error code.
1054                          */
1055                         tf->tf_scratch.gr8 = error;
1056                 }
1057         }
1058
1059         td->td_syscalls++;
1060
1061         /*
1062          * Check for misbehavior.
1063          */
1064         WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
1065             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
1066         KASSERT(td->td_critnest == 0,
1067             ("System call %s returning in a critical section",
1068             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"));
1069         KASSERT(td->td_locks == 0,
1070             ("System call %s returning with %d locks held",
1071             (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???",
1072             td->td_locks));
1073
1074         /*
1075          * Handle reschedule and other end-of-syscall issues
1076          */
1077         userret(td, tf);
1078
1079         CTR4(KTR_SYSC, "syscall exit thread %p pid %d proc %s code %d", td,
1080             td->td_proc->p_pid, td->td_proc->p_comm, code);
1081 #ifdef KTRACE
1082         if (KTRPOINT(td, KTR_SYSRET))
1083                 ktrsysret(code, error, td->td_retval[0]);
1084 #endif
1085
1086         /*
1087          * This works because errno is findable through the
1088          * register set.  If we ever support an emulation where this
1089          * is not the case, this code will need to be revisited.
1090          */
1091         STOPEVENT(p, S_SCX, code);
1092
1093         PTRACESTOP_SC(p, td, S_PT_SCX);
1094
1095         return (error);
1096 }