sys/amd64/amd64/trap.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (C) 1994, David Greenman
   5  * Copyright (c) 1990, 1993
   6  *      The Regents of the University of California.  All rights reserved.
   7  *
   8  * This code is derived from software contributed to Berkeley by
   9  * the University of Utah, and William Jolitz.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by the University of
  22  *      California, Berkeley and its contributors.
  23  * 4. Neither the name of the University nor the names of its contributors
  24  *    may be used to endorse or promote products derived from this software
  25  *    without specific prior written permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  37  * SUCH DAMAGE.
  38  *
  39  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 /*
  46  * AMD64 Trap and System call handling
  47  */
  48
  49 #include "opt_clock.h"
  50 #include "opt_compat.h"
  51 #include "opt_cpu.h"
  52 #include "opt_hwpmc_hooks.h"
  53 #include "opt_isa.h"
  54 #include "opt_kdb.h"
  55 #include "opt_stack.h"
  56
  57 #include <sys/param.h>
  58 #include <sys/bus.h>
  59 #include <sys/systm.h>
  60 #include <sys/proc.h>
  61 #include <sys/pioctl.h>
  62 #include <sys/ptrace.h>
  63 #include <sys/kdb.h>
  64 #include <sys/kernel.h>
  65 #include <sys/ktr.h>
  66 #include <sys/lock.h>
  67 #include <sys/mutex.h>
  68 #include <sys/resourcevar.h>
  69 #include <sys/signalvar.h>
  70 #include <sys/syscall.h>
  71 #include <sys/sysctl.h>
  72 #include <sys/sysent.h>
  73 #include <sys/uio.h>
  74 #include <sys/vmmeter.h>
  75 #ifdef HWPMC_HOOKS
  76 #include <sys/pmckern.h>
  77 PMC_SOFT_DEFINE( , , page_fault, all);
  78 PMC_SOFT_DEFINE( , , page_fault, read);
  79 PMC_SOFT_DEFINE( , , page_fault, write);
  80 #endif
  81
  82 #include <vm/vm.h>
  83 #include <vm/vm_param.h>
  84 #include <vm/pmap.h>
  85 #include <vm/vm_kern.h>
  86 #include <vm/vm_map.h>
  87 #include <vm/vm_page.h>
  88 #include <vm/vm_extern.h>
  89
  90 #include <machine/cpu.h>
  91 #include <machine/intr_machdep.h>
  92 #include <x86/mca.h>
  93 #include <machine/md_var.h>
  94 #include <machine/pcb.h>
  95 #ifdef SMP
  96 #include <machine/smp.h>
  97 #endif
  98 #include <machine/stack.h>
  99 #include <machine/trap.h>
 100 #include <machine/tss.h>
 101
 102 #ifdef KDTRACE_HOOKS
 103 #include <sys/dtrace_bsd.h>
 104 #endif
 105
 106 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
 107     IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
 108     IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
 109
 110 void __noinline trap(struct trapframe *frame);
 111 void trap_check(struct trapframe *frame);
 112 void dblfault_handler(struct trapframe *frame);
 113
 114 static int trap_pfault(struct trapframe *, int);
 115 static void trap_fatal(struct trapframe *, vm_offset_t);
 116
 117 #define MAX_TRAP_MSG            32
 118 static char *trap_msg[] = {
 119         "",                                     /*  0 unused */
 120         "privileged instruction fault",         /*  1 T_PRIVINFLT */
 121         "",                                     /*  2 unused */
 122         "breakpoint instruction fault",         /*  3 T_BPTFLT */
 123         "",                                     /*  4 unused */
 124         "",                                     /*  5 unused */
 125         "arithmetic trap",                      /*  6 T_ARITHTRAP */
 126         "",                                     /*  7 unused */
 127         "",                                     /*  8 unused */
 128         "general protection fault",             /*  9 T_PROTFLT */
 129         "debug exception",                      /* 10 T_TRCTRAP */
 130         "",                                     /* 11 unused */
 131         "page fault",                           /* 12 T_PAGEFLT */
 132         "",                                     /* 13 unused */
 133         "alignment fault",                      /* 14 T_ALIGNFLT */
 134         "",                                     /* 15 unused */
 135         "",                                     /* 16 unused */
 136         "",                                     /* 17 unused */
 137         "integer divide fault",                 /* 18 T_DIVIDE */
 138         "non-maskable interrupt trap",          /* 19 T_NMI */
 139         "overflow trap",                        /* 20 T_OFLOW */
 140         "FPU bounds check fault",               /* 21 T_BOUND */
 141         "FPU device not available",             /* 22 T_DNA */
 142         "double fault",                         /* 23 T_DOUBLEFLT */
 143         "FPU operand fetch fault",              /* 24 T_FPOPFLT */
 144         "invalid TSS fault",                    /* 25 T_TSSFLT */
 145         "segment not present fault",            /* 26 T_SEGNPFLT */
 146         "stack fault",                          /* 27 T_STKFLT */
 147         "machine check trap",                   /* 28 T_MCHK */
 148         "SIMD floating-point exception",        /* 29 T_XMMFLT */
 149         "reserved (unknown) fault",             /* 30 T_RESERVED */
 150         "",                                     /* 31 unused (reserved) */
 151         "DTrace pid return trap",               /* 32 T_DTRACE_RET */
 152 };
 153
 154 static int prot_fault_translation;
 155 SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN,
 156     &prot_fault_translation, 0,
 157     "Select signal to deliver on protection fault");
 158 static int uprintf_signal;
 159 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
 160     &uprintf_signal, 0,
 161     "Print debugging information on trap signal to ctty");
 162
 163 /*
 164  * Control L1D flush on return from NMI.
 165  *
 166  * Tunable  can be set to the following values:
 167  * 0 - only enable flush on return from NMI if required by vmm.ko (default)
 168  * >1 - always flush on return from NMI.
 169  *
 170  * Post-boot, the sysctl indicates if flushing is currently enabled.
 171  */
 172 int nmi_flush_l1d_sw;
 173 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
 174     &nmi_flush_l1d_sw, 0,
 175     "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
 176
 177 /*
 178  * Exception, fault, and trap interface to the FreeBSD kernel.
 179  * This common code is called from assembly language IDT gate entry
 180  * routines that prepare a suitable stack frame, and restore this
 181  * frame after the exception has been processed.
 182  */
 183
 184 void
 185 trap(struct trapframe *frame)
 186 {
 187         ksiginfo_t ksi;
 188         struct thread *td;
 189         struct proc *p;
 190         register_t addr, dr6;
 191         int signo, ucode;
 192         u_int type;
 193
 194         td = curthread;
 195         p = td->td_proc;
 196         signo = 0;
 197         ucode = 0;
 198         addr = 0;
 199         dr6 = 0;
 200
 201         VM_CNT_INC(v_trap);
 202         type = frame->tf_trapno;
 203
 204 #ifdef SMP
 205         /* Handler for NMI IPIs used for stopping CPUs. */
 206         if (type == T_NMI && ipi_nmi_handler() == 0)
 207                 return;
 208 #endif
 209
 210 #ifdef KDB
 211         if (kdb_active) {
 212                 kdb_reenter();
 213                 return;
 214         }
 215 #endif
 216
 217         if (type == T_RESERVED) {
 218                 trap_fatal(frame, 0);
 219                 return;
 220         }
 221
 222         if (type == T_NMI) {
 223 #ifdef HWPMC_HOOKS
 224                 /*
 225                  * CPU PMCs interrupt using an NMI.  If the PMC module is
 226                  * active, pass the 'rip' value to the PMC module's interrupt
 227                  * handler.  A non-zero return value from the handler means that
 228                  * the NMI was consumed by it and we can return immediately.
 229                  */
 230                 if (pmc_intr != NULL &&
 231                     (*pmc_intr)(frame) != 0)
 232                         return;
 233 #endif
 234
 235 #ifdef STACK
 236                 if (stack_nmi_handler(frame) != 0)
 237                         return;
 238 #endif
 239         }
 240
 241         if ((frame->tf_rflags & PSL_I) == 0) {
 242                 /*
 243                  * Buggy application or kernel code has disabled
 244                  * interrupts and then trapped.  Enabling interrupts
 245                  * now is wrong, but it is better than running with
 246                  * interrupts disabled until they are accidentally
 247                  * enabled later.
 248                  */
 249                 if (TRAPF_USERMODE(frame))
 250                         uprintf(
 251                             "pid %ld (%s): trap %d with interrupts disabled\n",
 252                             (long)curproc->p_pid, curthread->td_name, type);
 253                 else if (type != T_NMI && type != T_BPTFLT &&
 254                     type != T_TRCTRAP) {
 255                         /*
 256                          * XXX not quite right, since this may be for a
 257                          * multiple fault in user mode.
 258                          */
 259                         printf("kernel trap %d with interrupts disabled\n",
 260                             type);
 261
 262                         /*
 263                          * We shouldn't enable interrupts while holding a
 264                          * spin lock.
 265                          */
 266                         if (td->td_md.md_spinlock_count == 0)
 267                                 enable_intr();
 268                 }
 269         }
 270
 271         if (TRAPF_USERMODE(frame)) {
 272                 /* user trap */
 273
 274                 td->td_pticks = 0;
 275                 td->td_frame = frame;
 276                 addr = frame->tf_rip;
 277                 if (td->td_cowgen != p->p_cowgen)
 278                         thread_cow_update(td);
 279
 280                 switch (type) {
 281                 case T_PRIVINFLT:       /* privileged instruction fault */
 282                         signo = SIGILL;
 283                         ucode = ILL_PRVOPC;
 284                         break;
 285
 286                 case T_BPTFLT:          /* bpt instruction fault */
 287                         enable_intr();
 288 #ifdef KDTRACE_HOOKS
 289                         if (dtrace_pid_probe_ptr != NULL &&
 290                             dtrace_pid_probe_ptr(frame) == 0)
 291                                 return;
 292 #endif
 293                         signo = SIGTRAP;
 294                         ucode = TRAP_BRKPT;
 295                         break;
 296
 297                 case T_TRCTRAP:         /* debug exception */
 298                         enable_intr();
 299                         signo = SIGTRAP;
 300                         ucode = TRAP_TRACE;
 301                         dr6 = rdr6();
 302                         if ((dr6 & DBREG_DR6_BS) != 0) {
 303                                 PROC_LOCK(td->td_proc);
 304                                 if ((td->td_dbgflags & TDB_STEP) != 0) {
 305                                         td->td_frame->tf_rflags &= ~PSL_T;
 306                                         td->td_dbgflags &= ~TDB_STEP;
 307                                 }
 308                                 PROC_UNLOCK(td->td_proc);
 309                         }
 310                         break;
 311
 312                 case T_ARITHTRAP:       /* arithmetic trap */
 313                         ucode = fputrap_x87();
 314                         if (ucode == -1)
 315                                 return;
 316                         signo = SIGFPE;
 317                         break;
 318
 319                 case T_PROTFLT:         /* general protection fault */
 320                         signo = SIGBUS;
 321                         ucode = BUS_OBJERR;
 322                         break;
 323                 case T_STKFLT:          /* stack fault */
 324                 case T_SEGNPFLT:        /* segment not present fault */
 325                         signo = SIGBUS;
 326                         ucode = BUS_ADRERR;
 327                         break;
 328                 case T_TSSFLT:          /* invalid TSS fault */
 329                         signo = SIGBUS;
 330                         ucode = BUS_OBJERR;
 331                         break;
 332                 case T_ALIGNFLT:
 333                         signo = SIGBUS;
 334                         ucode = BUS_ADRALN;
 335                         break;
 336                 case T_DOUBLEFLT:       /* double fault */
 337                 default:
 338                         signo = SIGBUS;
 339                         ucode = BUS_OBJERR;
 340                         break;
 341
 342                 case T_PAGEFLT:         /* page fault */
 343                         /*
 344                          * Emulator can take care about this trap?
 345                          */
 346                         if (*p->p_sysent->sv_trap != NULL &&
 347                             (*p->p_sysent->sv_trap)(td) == 0)
 348                                 return;
 349
 350                         addr = frame->tf_addr;
 351                         signo = trap_pfault(frame, TRUE);
 352                         if (signo == -1)
 353                                 return;
 354                         if (signo == 0)
 355                                 goto userret;
 356                         if (signo == SIGSEGV) {
 357                                 ucode = SEGV_MAPERR;
 358                         } else if (prot_fault_translation == 0) {
 359                                 /*
 360                                  * Autodetect.  This check also covers
 361                                  * the images without the ABI-tag ELF
 362                                  * note.
 363                                  */
 364                                 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD &&
 365                                     p->p_osrel >= P_OSREL_SIGSEGV) {
 366                                         signo = SIGSEGV;
 367                                         ucode = SEGV_ACCERR;
 368                                 } else {
 369                                         signo = SIGBUS;
 370                                         ucode = T_PAGEFLT;
 371                                 }
 372                         } else if (prot_fault_translation == 1) {
 373                                 /*
 374                                  * Always compat mode.
 375                                  */
 376                                 signo = SIGBUS;
 377                                 ucode = T_PAGEFLT;
 378                         } else {
 379                                 /*
 380                                  * Always SIGSEGV mode.
 381                                  */
 382                                 signo = SIGSEGV;
 383                                 ucode = SEGV_ACCERR;
 384                         }
 385                         break;
 386
 387                 case T_DIVIDE:          /* integer divide fault */
 388                         ucode = FPE_INTDIV;
 389                         signo = SIGFPE;
 390                         break;
 391
 392 #ifdef DEV_ISA
 393                 case T_NMI:
 394                         nmi_handle_intr(type, frame);
 395                         return;
 396 #endif
 397
 398                 case T_OFLOW:           /* integer overflow fault */
 399                         ucode = FPE_INTOVF;
 400                         signo = SIGFPE;
 401                         break;
 402
 403                 case T_BOUND:           /* bounds check fault */
 404                         ucode = FPE_FLTSUB;
 405                         signo = SIGFPE;
 406                         break;
 407
 408                 case T_DNA:
 409                         /* transparent fault (due to context switch "late") */
 410                         KASSERT(PCB_USER_FPU(td->td_pcb),
 411                             ("kernel FPU ctx has leaked"));
 412                         fpudna();
 413                         return;
 414
 415                 case T_FPOPFLT:         /* FPU operand fetch fault */
 416                         ucode = ILL_COPROC;
 417                         signo = SIGILL;
 418                         break;
 419
 420                 case T_XMMFLT:          /* SIMD floating-point exception */
 421                         ucode = fputrap_sse();
 422                         if (ucode == -1)
 423                                 return;
 424                         signo = SIGFPE;
 425                         break;
 426 #ifdef KDTRACE_HOOKS
 427                 case T_DTRACE_RET:
 428                         enable_intr();
 429                         if (dtrace_return_probe_ptr != NULL)
 430                                 dtrace_return_probe_ptr(frame);
 431                         return;
 432 #endif
 433                 }
 434         } else {
 435                 /* kernel trap */
 436
 437                 KASSERT(cold || td->td_ucred != NULL,
 438                     ("kernel trap doesn't have ucred"));
 439                 switch (type) {
 440                 case T_PAGEFLT:                 /* page fault */
 441                         (void) trap_pfault(frame, FALSE);
 442                         return;
 443
 444                 case T_DNA:
 445                         if (PCB_USER_FPU(td->td_pcb))
 446                                 panic("Unregistered use of FPU in kernel");
 447                         fpudna();
 448                         return;
 449
 450                 case T_ARITHTRAP:       /* arithmetic trap */
 451                 case T_XMMFLT:          /* SIMD floating-point exception */
 452                 case T_FPOPFLT:         /* FPU operand fetch fault */
 453                         /*
 454                          * For now, supporting kernel handler
 455                          * registration for FPU traps is overkill.
 456                          */
 457                         trap_fatal(frame, 0);
 458                         return;
 459
 460                 case T_STKFLT:          /* stack fault */
 461                 case T_PROTFLT:         /* general protection fault */
 462                 case T_SEGNPFLT:        /* segment not present fault */
 463                         if (td->td_intr_nesting_level != 0)
 464                                 break;
 465
 466                         /*
 467                          * Invalid segment selectors and out of bounds
 468                          * %rip's and %rsp's can be set up in user mode.
 469                          * This causes a fault in kernel mode when the
 470                          * kernel tries to return to user mode.  We want
 471                          * to get this fault so that we can fix the
 472                          * problem here and not have to check all the
 473                          * selectors and pointers when the user changes
 474                          * them.
 475                          *
 476                          * In case of PTI, the IRETQ faulted while the
 477                          * kernel used the pti stack, and exception
 478                          * frame records %rsp value pointing to that
 479                          * stack.  If we return normally to
 480                          * doreti_iret_fault, the trapframe is
 481                          * reconstructed on pti stack, and calltrap()
 482                          * called on it as well.  Due to the very
 483                          * limited pti stack size, kernel does not
 484                          * survive for too long.  Switch to the normal
 485                          * thread stack for the trap handling.
 486                          *
 487                          * Magic '5' is the number of qwords occupied by
 488                          * the hardware trap frame.
 489                          */
 490                         if (frame->tf_rip == (long)doreti_iret) {
 491                                 frame->tf_rip = (long)doreti_iret_fault;
 492                                 if ((PCPU_GET(curpmap)->pm_ucr3 !=
 493                                     PMAP_NO_CR3) &&
 494                                     (frame->tf_rsp == (uintptr_t)PCPU_GET(
 495                                     pti_rsp0) - 5 * sizeof(register_t))) {
 496                                         frame->tf_rsp = PCPU_GET(rsp0) - 5 *
 497                                             sizeof(register_t);
 498                                 }
 499                                 return;
 500                         }
 501                         if (frame->tf_rip == (long)ld_ds) {
 502                                 frame->tf_rip = (long)ds_load_fault;
 503                                 return;
 504                         }
 505                         if (frame->tf_rip == (long)ld_es) {
 506                                 frame->tf_rip = (long)es_load_fault;
 507                                 return;
 508                         }
 509                         if (frame->tf_rip == (long)ld_fs) {
 510                                 frame->tf_rip = (long)fs_load_fault;
 511                                 return;
 512                         }
 513                         if (frame->tf_rip == (long)ld_gs) {
 514                                 frame->tf_rip = (long)gs_load_fault;
 515                                 return;
 516                         }
 517                         if (frame->tf_rip == (long)ld_gsbase) {
 518                                 frame->tf_rip = (long)gsbase_load_fault;
 519                                 return;
 520                         }
 521                         if (frame->tf_rip == (long)ld_fsbase) {
 522                                 frame->tf_rip = (long)fsbase_load_fault;
 523                                 return;
 524                         }
 525                         if (curpcb->pcb_onfault != NULL) {
 526                                 frame->tf_rip = (long)curpcb->pcb_onfault;
 527                                 return;
 528                         }
 529                         break;
 530
 531                 case T_TSSFLT:
 532                         /*
 533                          * PSL_NT can be set in user mode and isn't cleared
 534                          * automatically when the kernel is entered.  This
 535                          * causes a TSS fault when the kernel attempts to
 536                          * `iret' because the TSS link is uninitialized.  We
 537                          * want to get this fault so that we can fix the
 538                          * problem here and not every time the kernel is
 539                          * entered.
 540                          */
 541                         if (frame->tf_rflags & PSL_NT) {
 542                                 frame->tf_rflags &= ~PSL_NT;
 543                                 return;
 544                         }
 545                         break;
 546
 547                 case T_TRCTRAP:  /* debug exception */
 548                         /* Clear any pending debug events. */
 549                         dr6 = rdr6();
 550                         load_dr6(0);
 551
 552                         /*
 553                          * Ignore debug register exceptions due to
 554                          * accesses in the user's address space, which
 555                          * can happen under several conditions such as
 556                          * if a user sets a watchpoint on a buffer and
 557                          * then passes that buffer to a system call.
 558                          * We still want to get TRCTRAPS for addresses
 559                          * in kernel space because that is useful when
 560                          * debugging the kernel.
 561                          */
 562                         if (user_dbreg_trap(dr6))
 563                                 return;
 564
 565                         /*
 566                          * Malicious user code can configure a debug
 567                          * register watchpoint to trap on data access
 568                          * to the top of stack and then execute 'pop
 569                          * %ss; int 3'.  Due to exception deferral for
 570                          * 'pop %ss', the CPU will not interrupt 'int
 571                          * 3' to raise the DB# exception for the debug
 572                          * register but will postpone the DB# until
 573                          * execution of the first instruction of the
 574                          * BP# handler (in kernel mode).  Normally the
 575                          * previous check would ignore DB# exceptions
 576                          * for watchpoints on user addresses raised in
 577                          * kernel mode.  However, some CPU errata
 578                          * include cases where DB# exceptions do not
 579                          * properly set bits in %dr6, e.g. Haswell
 580                          * HSD23 and Skylake-X SKZ24.
 581                          *
 582                          * A deferred DB# can also be raised on the
 583                          * first instructions of system call entry
 584                          * points or single-step traps via similar use
 585                          * of 'pop %ss' or 'mov xxx, %ss'.
 586                          */
 587                         if (pti) {
 588                                 if (frame->tf_rip ==
 589                                     (uintptr_t)IDTVEC(fast_syscall_pti) ||
 590 #ifdef COMPAT_FREEBSD32
 591                                     frame->tf_rip ==
 592                                     (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
 593 #endif
 594                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
 595                                         return;
 596                         } else {
 597                                 if (frame->tf_rip ==
 598                                     (uintptr_t)IDTVEC(fast_syscall) ||
 599 #ifdef COMPAT_FREEBSD32
 600                                     frame->tf_rip ==
 601                                     (uintptr_t)IDTVEC(int0x80_syscall) ||
 602 #endif
 603                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt))
 604                                         return;
 605                         }
 606                         if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
 607                             /* Needed for AMD. */
 608                             frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
 609                                 return;
 610                         /*
 611                          * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
 612                          */
 613                 case T_BPTFLT:
 614                         /*
 615                          * If KDB is enabled, let it handle the debugger trap.
 616                          * Otherwise, debugger traps "can't happen".
 617                          */
 618 #ifdef KDB
 619                         if (kdb_trap(type, dr6, frame))
 620                                 return;
 621 #endif
 622                         break;
 623
 624 #ifdef DEV_ISA
 625                 case T_NMI:
 626                         nmi_handle_intr(type, frame);
 627                         return;
 628 #endif
 629                 }
 630
 631                 trap_fatal(frame, 0);
 632                 return;
 633         }
 634
 635         /* Translate fault for emulators (e.g. Linux) */
 636         if (*p->p_sysent->sv_transtrap != NULL)
 637                 signo = (*p->p_sysent->sv_transtrap)(signo, type);
 638
 639         ksiginfo_init_trap(&ksi);
 640         ksi.ksi_signo = signo;
 641         ksi.ksi_code = ucode;
 642         ksi.ksi_trapno = type;
 643         ksi.ksi_addr = (void *)addr;
 644         if (uprintf_signal) {
 645                 uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
 646                     "addr 0x%lx rsp 0x%lx rip 0x%lx "
 647                     "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
 648                     p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
 649                     addr, frame->tf_rsp, frame->tf_rip,
 650                     fubyte((void *)(frame->tf_rip + 0)),
 651                     fubyte((void *)(frame->tf_rip + 1)),
 652                     fubyte((void *)(frame->tf_rip + 2)),
 653                     fubyte((void *)(frame->tf_rip + 3)),
 654                     fubyte((void *)(frame->tf_rip + 4)),
 655                     fubyte((void *)(frame->tf_rip + 5)),
 656                     fubyte((void *)(frame->tf_rip + 6)),
 657                     fubyte((void *)(frame->tf_rip + 7)));
 658         }
 659         KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
 660         trapsignal(td, &ksi);
 661
 662 userret:
 663         userret(td, frame);
 664         KASSERT(PCB_USER_FPU(td->td_pcb),
 665             ("Return from trap with kernel FPU ctx leaked"));
 666 }
 667
 668 /*
 669  * Ensure that we ignore any DTrace-induced faults. This function cannot
 670  * be instrumented, so it cannot generate such faults itself.
 671  */
 672 void
 673 trap_check(struct trapframe *frame)
 674 {
 675
 676 #ifdef KDTRACE_HOOKS
 677         if (dtrace_trap_func != NULL &&
 678             (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
 679                 return;
 680 #endif
 681         trap(frame);
 682 }
 683
 684 static bool
 685 trap_is_smap(struct trapframe *frame)
 686 {
 687
 688         /*
 689          * A page fault on a userspace address is classified as
 690          * SMAP-induced if:
 691          * - SMAP is supported;
 692          * - kernel mode accessed present data page;
 693          * - rflags.AC was cleared.
 694          * Kernel must never access user space with rflags.AC cleared
 695          * if SMAP is enabled.
 696          */
 697         return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
 698             (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
 699             PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
 700 }
 701
 702 static bool
 703 trap_is_pti(struct trapframe *frame)
 704 {
 705
 706         return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
 707             pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
 708             PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
 709             (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
 710             (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
 711 }
 712
 713 static int
 714 trap_pfault(struct trapframe *frame, int usermode)
 715 {
 716         struct thread *td;
 717         struct proc *p;
 718         vm_map_t map;
 719         vm_offset_t va;
 720         int rv;
 721         vm_prot_t ftype;
 722         vm_offset_t eva;
 723
 724         td = curthread;
 725         p = td->td_proc;
 726         eva = frame->tf_addr;
 727
 728         if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
 729                 /*
 730                  * Due to both processor errata and lazy TLB invalidation when
 731                  * access restrictions are removed from virtual pages, memory
 732                  * accesses that are allowed by the physical mapping layer may
 733                  * nonetheless cause one spurious page fault per virtual page.
 734                  * When the thread is executing a "no faulting" section that
 735                  * is bracketed by vm_fault_{disable,enable}_pagefaults(),
 736                  * every page fault is treated as a spurious page fault,
 737                  * unless it accesses the same virtual address as the most
 738                  * recent page fault within the same "no faulting" section.
 739                  */
 740                 if (td->td_md.md_spurflt_addr != eva ||
 741                     (td->td_pflags & TDP_RESETSPUR) != 0) {
 742                         /*
 743                          * Do nothing to the TLB.  A stale TLB entry is
 744                          * flushed automatically by a page fault.
 745                          */
 746                         td->td_md.md_spurflt_addr = eva;
 747                         td->td_pflags &= ~TDP_RESETSPUR;
 748                         return (0);
 749                 }
 750         } else {
 751                 /*
 752                  * If we get a page fault while in a critical section, then
 753                  * it is most likely a fatal kernel page fault.  The kernel
 754                  * is already going to panic trying to get a sleep lock to
 755                  * do the VM lookup, so just consider it a fatal trap so the
 756                  * kernel can print out a useful trap message and even get
 757                  * to the debugger.
 758                  *
 759                  * If we get a page fault while holding a non-sleepable
 760                  * lock, then it is most likely a fatal kernel page fault.
 761                  * If WITNESS is enabled, then it's going to whine about
 762                  * bogus LORs with various VM locks, so just skip to the
 763                  * fatal trap handling directly.
 764                  */
 765                 if (td->td_critnest != 0 ||
 766                     WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
 767                     "Kernel page fault") != 0) {
 768                         trap_fatal(frame, eva);
 769                         return (-1);
 770                 }
 771         }
 772         va = trunc_page(eva);
 773         if (va >= VM_MIN_KERNEL_ADDRESS) {
 774                 /*
 775                  * Don't allow user-mode faults in kernel address space.
 776                  */
 777                 if (usermode)
 778                         return (SIGSEGV);
 779
 780                 map = kernel_map;
 781         } else {
 782                 map = &p->p_vmspace->vm_map;
 783
 784                 /*
 785                  * When accessing a usermode address, kernel must be
 786                  * ready to accept the page fault, and provide a
 787                  * handling routine.  Since accessing the address
 788                  * without the handler is a bug, do not try to handle
 789                  * it normally, and panic immediately.
 790                  *
 791                  * If SMAP is enabled, filter SMAP faults also,
 792                  * because illegal access might occur to the mapped
 793                  * user address, causing infinite loop.
 794                  */
 795                 if (!usermode && (td->td_intr_nesting_level != 0 ||
 796                     trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
 797                         trap_fatal(frame, eva);
 798                         return (-1);
 799                 }
 800         }
 801
 802         /*
 803          * If the trap was caused by errant bits in the PTE then panic.
 804          */
 805         if (frame->tf_err & PGEX_RSV) {
 806                 trap_fatal(frame, eva);
 807                 return (-1);
 808         }
 809
 810         /*
 811          * User-mode protection key violation (PKU).  May happen
 812          * either from usermode or from kernel if copyin accessed
 813          * key-protected mapping.
 814          */
 815         if ((frame->tf_err & PGEX_PK) != 0) {
 816                 if (eva > VM_MAXUSER_ADDRESS) {
 817                         trap_fatal(frame, eva);
 818                         return (-1);
 819                 }
 820                 rv = KERN_PROTECTION_FAILURE;
 821                 goto after_vmfault;
 822         }
 823
 824         /*
 825          * If nx protection of the usermode portion of kernel page
 826          * tables caused trap, panic.
 827          */
 828         if (usermode && trap_is_pti(frame))
 829                 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
 830                     p->p_comm, frame->tf_err);
 831
 832         /*
 833          * PGEX_I is defined only if the execute disable bit capability is
 834          * supported and enabled.
 835          */
 836         if (frame->tf_err & PGEX_W)
 837                 ftype = VM_PROT_WRITE;
 838         else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
 839                 ftype = VM_PROT_EXECUTE;
 840         else
 841                 ftype = VM_PROT_READ;
 842
 843         /* Fault in the page. */
 844         rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
 845         if (rv == KERN_SUCCESS) {
 846 #ifdef HWPMC_HOOKS
 847                 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
 848                         PMC_SOFT_CALL_TF( , , page_fault, all, frame);
 849                         if (ftype == VM_PROT_READ)
 850                                 PMC_SOFT_CALL_TF( , , page_fault, read,
 851                                     frame);
 852                         else
 853                                 PMC_SOFT_CALL_TF( , , page_fault, write,
 854                                     frame);
 855                 }
 856 #endif
 857                 return (0);
 858         }
 859 after_vmfault:
 860         if (!usermode) {
 861                 if (td->td_intr_nesting_level == 0 &&
 862                     curpcb->pcb_onfault != NULL) {
 863                         frame->tf_rip = (long)curpcb->pcb_onfault;
 864                         return (0);
 865                 }
 866                 trap_fatal(frame, eva);
 867                 return (-1);
 868         }
 869         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 870 }
 871
 872 static void
 873 trap_fatal(frame, eva)
 874         struct trapframe *frame;
 875         vm_offset_t eva;
 876 {
 877         int code, ss;
 878         u_int type;
 879         struct soft_segment_descriptor softseg;
 880         char *msg;
 881 #ifdef KDB
 882         bool handled;
 883 #endif
 884
 885         code = frame->tf_err;
 886         type = frame->tf_trapno;
 887         sdtossd(&gdt[NGDT * PCPU_GET(cpuid) + IDXSEL(frame->tf_cs & 0xffff)],
 888             &softseg);
 889
 890         if (type <= MAX_TRAP_MSG)
 891                 msg = trap_msg[type];
 892         else
 893                 msg = "UNKNOWN";
 894         printf("\n\nFatal trap %d: %s while in %s mode\n", type, msg,
 895             TRAPF_USERMODE(frame) ? "user" : "kernel");
 896 #ifdef SMP
 897         /* two separate prints in case of a trap on an unmapped page */
 898         printf("cpuid = %d; ", PCPU_GET(cpuid));
 899         printf("apic id = %02x\n", PCPU_GET(apic_id));
 900 #endif
 901         if (type == T_PAGEFLT) {
 902                 printf("fault virtual address   = 0x%lx\n", eva);
 903                 printf("fault code              = %s %s %s%s%s, %s\n",
 904                         code & PGEX_U ? "user" : "supervisor",
 905                         code & PGEX_W ? "write" : "read",
 906                         code & PGEX_I ? "instruction" : "data",
 907                         code & PGEX_PK ? " prot key" : " ",
 908                         code & PGEX_SGX ? " SGX" : " ",
 909                         code & PGEX_RSV ? "reserved bits in PTE" :
 910                         code & PGEX_P ? "protection violation" : "page not present");
 911         }
 912         printf("instruction pointer     = 0x%lx:0x%lx\n",
 913                frame->tf_cs & 0xffff, frame->tf_rip);
 914         ss = frame->tf_ss & 0xffff;
 915         printf("stack pointer           = 0x%x:0x%lx\n", ss, frame->tf_rsp);
 916         printf("frame pointer           = 0x%x:0x%lx\n", ss, frame->tf_rbp);
 917         printf("code segment            = base 0x%lx, limit 0x%lx, type 0x%x\n",
 918                softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
 919         printf("                        = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
 920                softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
 921                softseg.ssd_gran);
 922         printf("processor eflags        = ");
 923         if (frame->tf_rflags & PSL_T)
 924                 printf("trace trap, ");
 925         if (frame->tf_rflags & PSL_I)
 926                 printf("interrupt enabled, ");
 927         if (frame->tf_rflags & PSL_NT)
 928                 printf("nested task, ");
 929         if (frame->tf_rflags & PSL_RF)
 930                 printf("resume, ");
 931         printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
 932         printf("current process         = %d (%s)\n",
 933             curproc->p_pid, curthread->td_name);
 934
 935 #ifdef KDB
 936         if (debugger_on_trap) {
 937                 kdb_why = KDB_WHY_TRAP;
 938                 handled = kdb_trap(type, 0, frame);
 939                 kdb_why = KDB_WHY_UNSET;
 940                 if (handled)
 941                         return;
 942         }
 943 #endif
 944         printf("trap number             = %d\n", type);
 945         if (type <= MAX_TRAP_MSG)
 946                 panic("%s", trap_msg[type]);
 947         else
 948                 panic("unknown/reserved trap");
 949 }
 950
 951 /*
 952  * Double fault handler. Called when a fault occurs while writing
 953  * a frame for a trap/exception onto the stack. This usually occurs
 954  * when the stack overflows (such is the case with infinite recursion,
 955  * for example).
 956  */
 957 void
 958 dblfault_handler(struct trapframe *frame)
 959 {
 960 #ifdef KDTRACE_HOOKS
 961         if (dtrace_doubletrap_func != NULL)
 962                 (*dtrace_doubletrap_func)();
 963 #endif
 964         printf("\nFatal double fault\n"
 965             "rip %#lx rsp %#lx rbp %#lx\n"
 966             "rax %#lx rdx %#lx rbx %#lx\n"
 967             "rcx %#lx rsi %#lx rdi %#lx\n"
 968             "r8 %#lx r9 %#lx r10 %#lx\n"
 969             "r11 %#lx r12 %#lx r13 %#lx\n"
 970             "r14 %#lx r15 %#lx rflags %#lx\n"
 971             "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
 972             "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
 973             frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
 974             frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
 975             frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
 976             frame->tf_r8, frame->tf_r9, frame->tf_r10,
 977             frame->tf_r11, frame->tf_r12, frame->tf_r13,
 978             frame->tf_r14, frame->tf_r15, frame->tf_rflags,
 979             frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
 980             frame->tf_fs, frame->tf_gs,
 981             rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
 982 #ifdef SMP
 983         /* two separate prints in case of a trap on an unmapped page */
 984         printf("cpuid = %d; ", PCPU_GET(cpuid));
 985         printf("apic id = %02x\n", PCPU_GET(apic_id));
 986 #endif
 987         panic("double fault");
 988 }
 989
 990 static int __noinline
 991 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
 992 {
 993         struct proc *p;
 994         struct trapframe *frame;
 995         register_t *argp;
 996         caddr_t params;
 997         int reg, regcnt, error;
 998
 999         p = td->td_proc;
1000         frame = td->td_frame;
1001         reg = 0;
1002         regcnt = NARGREGS;
1003
1004         sa->code = frame->tf_rax;
1005
1006         if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
1007                 sa->code = frame->tf_rdi;
1008                 reg++;
1009                 regcnt--;
1010         }
1011
1012         if (sa->code >= p->p_sysent->sv_size)
1013                 sa->callp = &p->p_sysent->sv_table[0];
1014         else
1015                 sa->callp = &p->p_sysent->sv_table[sa->code];
1016
1017         sa->narg = sa->callp->sy_narg;
1018         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
1019         argp = &frame->tf_rdi;
1020         argp += reg;
1021         memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
1022         if (sa->narg > regcnt) {
1023                 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
1024                 error = copyin(params, &sa->args[regcnt],
1025                     (sa->narg - regcnt) * sizeof(sa->args[0]));
1026                 if (__predict_false(error != 0))
1027                         return (error);
1028         }
1029
1030         td->td_retval[0] = 0;
1031         td->td_retval[1] = frame->tf_rdx;
1032
1033         return (0);
1034 }
1035
1036 int
1037 cpu_fetch_syscall_args(struct thread *td)
1038 {
1039         struct proc *p;
1040         struct trapframe *frame;
1041         struct syscall_args *sa;
1042
1043         p = td->td_proc;
1044         frame = td->td_frame;
1045         sa = &td->td_sa;
1046
1047         sa->code = frame->tf_rax;
1048
1049         if (__predict_false(sa->code == SYS_syscall ||
1050             sa->code == SYS___syscall ||
1051             sa->code >= p->p_sysent->sv_size))
1052                 return (cpu_fetch_syscall_args_fallback(td, sa));
1053
1054         sa->callp = &p->p_sysent->sv_table[sa->code];
1055         sa->narg = sa->callp->sy_narg;
1056         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
1057
1058         if (__predict_false(sa->narg > NARGREGS))
1059                 return (cpu_fetch_syscall_args_fallback(td, sa));
1060
1061         memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
1062
1063         td->td_retval[0] = 0;
1064         td->td_retval[1] = frame->tf_rdx;
1065
1066         return (0);
1067 }
1068
1069 #include "../../kern/subr_syscall.c"
1070
1071 static void (*syscall_ret_l1d_flush)(void);
1072 int syscall_ret_l1d_flush_mode;
1073
1074 static void
1075 flush_l1d_hw(void)
1076 {
1077
1078         wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
1079 }
1080
1081 static void __inline
1082 amd64_syscall_ret_flush_l1d_inline(int error)
1083 {
1084         void (*p)(void);
1085
1086         if (error != 0 && error != EEXIST && error != EAGAIN &&
1087             error != EXDEV && error != ENOENT && error != ENOTCONN &&
1088             error != EINPROGRESS) {
1089                 p = syscall_ret_l1d_flush;
1090                 if (p != NULL)
1091                         p();
1092         }
1093 }
1094
1095 void
1096 amd64_syscall_ret_flush_l1d(int error)
1097 {
1098
1099         amd64_syscall_ret_flush_l1d_inline(error);
1100 }
1101
1102 void
1103 amd64_syscall_ret_flush_l1d_recalc(void)
1104 {
1105         bool l1d_hw;
1106
1107         l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
1108 again:
1109         switch (syscall_ret_l1d_flush_mode) {
1110         case 0:
1111                 syscall_ret_l1d_flush = NULL;
1112                 break;
1113         case 1:
1114                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
1115                     flush_l1d_sw_abi;
1116                 break;
1117         case 2:
1118                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
1119                 break;
1120         case 3:
1121                 syscall_ret_l1d_flush = flush_l1d_sw_abi;
1122                 break;
1123         default:
1124                 syscall_ret_l1d_flush_mode = 1;
1125                 goto again;
1126         }
1127 }
1128
1129 static int
1130 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
1131 {
1132         int error, val;
1133
1134         val = syscall_ret_l1d_flush_mode;
1135         error = sysctl_handle_int(oidp, &val, 0, req);
1136         if (error != 0 || req->newptr == NULL)
1137                 return (error);
1138         syscall_ret_l1d_flush_mode = val;
1139         amd64_syscall_ret_flush_l1d_recalc();
1140         return (0);
1141 }
1142 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
1143     CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
1144     machdep_syscall_ret_flush_l1d, "I",
1145     "Flush L1D on syscall return with error (0 - off, 1 - on, "
1146     "2 - use hw only, 3 - use sw only");
1147
1148
1149 /*
1150  * System call handler for native binaries.  The trap frame is already
1151  * set up by the assembler trampoline and a pointer to it is saved in
1152  * td_frame.
1153  */
1154 void
1155 amd64_syscall(struct thread *td, int traced)
1156 {
1157         int error;
1158         ksiginfo_t ksi;
1159
1160 #ifdef DIAGNOSTIC
1161         if (!TRAPF_USERMODE(td->td_frame)) {
1162                 panic("syscall");
1163                 /* NOT REACHED */
1164         }
1165 #endif
1166         error = syscallenter(td);
1167
1168         /*
1169          * Traced syscall.
1170          */
1171         if (__predict_false(traced)) {
1172                 td->td_frame->tf_rflags &= ~PSL_T;
1173                 ksiginfo_init_trap(&ksi);
1174                 ksi.ksi_signo = SIGTRAP;
1175                 ksi.ksi_code = TRAP_TRACE;
1176                 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
1177                 trapsignal(td, &ksi);
1178         }
1179
1180         KASSERT(PCB_USER_FPU(td->td_pcb),
1181             ("System call %s returning with kernel FPU ctx leaked",
1182              syscallname(td->td_proc, td->td_sa.code)));
1183         KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
1184             ("System call %s returning with mangled pcb_save",
1185              syscallname(td->td_proc, td->td_sa.code)));
1186         KASSERT(td->td_md.md_invl_gen.gen == 0,
1187             ("System call %s returning with leaked invl_gen %lu",
1188             syscallname(td->td_proc, td->td_sa.code),
1189             td->td_md.md_invl_gen.gen));
1190
1191         syscallret(td, error);
1192
1193         /*
1194          * If the user-supplied value of %rip is not a canonical
1195          * address, then some CPUs will trigger a ring 0 #GP during
1196          * the sysret instruction.  However, the fault handler would
1197          * execute in ring 0 with the user's %gs and %rsp which would
1198          * not be safe.  Instead, use the full return path which
1199          * catches the problem safely.
1200          */
1201         if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
1202                 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
1203
1204         amd64_syscall_ret_flush_l1d_inline(error);
1205 }