sys/amd64/amd64/trap.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (C) 1994, David Greenman
   5  * Copyright (c) 1990, 1993
   6  *      The Regents of the University of California.  All rights reserved.
   7  *
   8  * This code is derived from software contributed to Berkeley by
   9  * the University of Utah, and William Jolitz.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by the University of
  22  *      California, Berkeley and its contributors.
  23  * 4. Neither the name of the University nor the names of its contributors
  24  *    may be used to endorse or promote products derived from this software
  25  *    without specific prior written permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  37  * SUCH DAMAGE.
  38  *
  39  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 /*
  46  * AMD64 Trap and System call handling
  47  */
  48
  49 #include "opt_clock.h"
  50 #include "opt_compat.h"
  51 #include "opt_cpu.h"
  52 #include "opt_hwpmc_hooks.h"
  53 #include "opt_isa.h"
  54 #include "opt_kdb.h"
  55
  56 #include <sys/param.h>
  57 #include <sys/asan.h>
  58 #include <sys/bus.h>
  59 #include <sys/systm.h>
  60 #include <sys/proc.h>
  61 #include <sys/ptrace.h>
  62 #include <sys/kdb.h>
  63 #include <sys/kernel.h>
  64 #include <sys/ktr.h>
  65 #include <sys/lock.h>
  66 #include <sys/mutex.h>
  67 #include <sys/resourcevar.h>
  68 #include <sys/signalvar.h>
  69 #include <sys/syscall.h>
  70 #include <sys/sysctl.h>
  71 #include <sys/sysent.h>
  72 #include <sys/uio.h>
  73 #include <sys/vmmeter.h>
  74 #ifdef HWPMC_HOOKS
  75 #include <sys/pmckern.h>
  76 PMC_SOFT_DEFINE( , , page_fault, all);
  77 PMC_SOFT_DEFINE( , , page_fault, read);
  78 PMC_SOFT_DEFINE( , , page_fault, write);
  79 #endif
  80
  81 #include <vm/vm.h>
  82 #include <vm/vm_param.h>
  83 #include <vm/pmap.h>
  84 #include <vm/vm_kern.h>
  85 #include <vm/vm_map.h>
  86 #include <vm/vm_page.h>
  87 #include <vm/vm_extern.h>
  88
  89 #include <machine/cpu.h>
  90 #include <machine/intr_machdep.h>
  91 #include <x86/mca.h>
  92 #include <machine/md_var.h>
  93 #include <machine/pcb.h>
  94 #ifdef SMP
  95 #include <machine/smp.h>
  96 #endif
  97 #include <machine/stack.h>
  98 #include <machine/trap.h>
  99 #include <machine/tss.h>
 100
 101 #ifdef KDTRACE_HOOKS
 102 #include <sys/dtrace_bsd.h>
 103 #endif
 104
 105 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
 106     IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
 107     IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
 108
 109 void __noinline trap(struct trapframe *frame);
 110 void trap_check(struct trapframe *frame);
 111 void dblfault_handler(struct trapframe *frame);
 112
 113 static int trap_pfault(struct trapframe *, bool, int *, int *);
 114 static void trap_fatal(struct trapframe *, vm_offset_t);
 115 #ifdef KDTRACE_HOOKS
 116 static bool trap_user_dtrace(struct trapframe *,
 117     int (**hook)(struct trapframe *));
 118 #endif
 119
 120 static const char UNKNOWN[] = "unknown";
 121 static const char *const trap_msg[] = {
 122         [0] =                   UNKNOWN,                        /* unused */
 123         [T_PRIVINFLT] =         "privileged instruction fault",
 124         [2] =                   UNKNOWN,                        /* unused */
 125         [T_BPTFLT] =            "breakpoint instruction fault",
 126         [4] =                   UNKNOWN,                        /* unused */
 127         [5] =                   UNKNOWN,                        /* unused */
 128         [T_ARITHTRAP] =         "arithmetic trap",
 129         [7] =                   UNKNOWN,                        /* unused */
 130         [8] =                   UNKNOWN,                        /* unused */
 131         [T_PROTFLT] =           "general protection fault",
 132         [T_TRCTRAP] =           "debug exception",
 133         [11] =                  UNKNOWN,                        /* unused */
 134         [T_PAGEFLT] =           "page fault",
 135         [13] =                  UNKNOWN,                        /* unused */
 136         [T_ALIGNFLT] =          "alignment fault",
 137         [15] =                  UNKNOWN,                        /* unused */
 138         [16] =                  UNKNOWN,                        /* unused */
 139         [17] =                  UNKNOWN,                        /* unused */
 140         [T_DIVIDE] =            "integer divide fault",
 141         [T_NMI] =               "non-maskable interrupt trap",
 142         [T_OFLOW] =             "overflow trap",
 143         [T_BOUND] =             "FPU bounds check fault",
 144         [T_DNA] =               "FPU device not available",
 145         [T_DOUBLEFLT] =         "double fault",
 146         [T_FPOPFLT] =           "FPU operand fetch fault",
 147         [T_TSSFLT] =            "invalid TSS fault",
 148         [T_SEGNPFLT] =          "segment not present fault",
 149         [T_STKFLT] =            "stack fault",
 150         [T_MCHK] =              "machine check trap",
 151         [T_XMMFLT] =            "SIMD floating-point exception",
 152         [T_RESERVED] =          "reserved (unknown) fault",
 153         [31] =                  UNKNOWN,                        /* reserved */
 154         [T_DTRACE_RET] =        "DTrace pid return trap",
 155 };
 156
 157 static int uprintf_signal;
 158 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
 159     &uprintf_signal, 0,
 160     "Print debugging information on trap signal to ctty");
 161
 162 /*
 163  * Control L1D flush on return from NMI.
 164  *
 165  * Tunable  can be set to the following values:
 166  * 0 - only enable flush on return from NMI if required by vmm.ko (default)
 167  * >1 - always flush on return from NMI.
 168  *
 169  * Post-boot, the sysctl indicates if flushing is currently enabled.
 170  */
 171 int nmi_flush_l1d_sw;
 172 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
 173     &nmi_flush_l1d_sw, 0,
 174     "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
 175
 176 /*
 177  * Table of handlers for various segment load faults.
 178  */
 179 static const struct {
 180         uintptr_t       faddr;
 181         uintptr_t       fhandler;
 182 } sfhandlers[] = {
 183         {
 184                 .faddr = (uintptr_t)ld_ds,
 185                 .fhandler = (uintptr_t)ds_load_fault,
 186         },
 187         {
 188                 .faddr = (uintptr_t)ld_es,
 189                 .fhandler = (uintptr_t)es_load_fault,
 190         },
 191         {
 192                 .faddr = (uintptr_t)ld_fs,
 193                 .fhandler = (uintptr_t)fs_load_fault,
 194         },
 195         {
 196                 .faddr = (uintptr_t)ld_gs,
 197                 .fhandler = (uintptr_t)gs_load_fault,
 198         },
 199         {
 200                 .faddr = (uintptr_t)ld_gsbase,
 201                 .fhandler = (uintptr_t)gsbase_load_fault
 202         },
 203         {
 204                 .faddr = (uintptr_t)ld_fsbase,
 205                 .fhandler = (uintptr_t)fsbase_load_fault,
 206         },
 207 };
 208
 209 /*
 210  * Exception, fault, and trap interface to the FreeBSD kernel.
 211  * This common code is called from assembly language IDT gate entry
 212  * routines that prepare a suitable stack frame, and restore this
 213  * frame after the exception has been processed.
 214  */
 215
 216 void
 217 trap(struct trapframe *frame)
 218 {
 219         ksiginfo_t ksi;
 220         struct thread *td;
 221         struct proc *p;
 222         register_t addr, dr6;
 223         size_t i;
 224         int pf, signo, ucode;
 225         u_int type;
 226
 227         td = curthread;
 228         p = td->td_proc;
 229         dr6 = 0;
 230
 231         kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
 232
 233         VM_CNT_INC(v_trap);
 234         type = frame->tf_trapno;
 235
 236 #ifdef SMP
 237         /* Handler for NMI IPIs used for stopping CPUs. */
 238         if (type == T_NMI && ipi_nmi_handler() == 0)
 239                 return;
 240 #endif
 241
 242 #ifdef KDB
 243         if (kdb_active) {
 244                 kdb_reenter();
 245                 return;
 246         }
 247 #endif
 248
 249         if (type == T_RESERVED) {
 250                 trap_fatal(frame, 0);
 251                 return;
 252         }
 253
 254         if (type == T_NMI) {
 255 #ifdef HWPMC_HOOKS
 256                 /*
 257                  * CPU PMCs interrupt using an NMI.  If the PMC module is
 258                  * active, pass the 'rip' value to the PMC module's interrupt
 259                  * handler.  A non-zero return value from the handler means that
 260                  * the NMI was consumed by it and we can return immediately.
 261                  */
 262                 if (pmc_intr != NULL &&
 263                     (*pmc_intr)(frame) != 0)
 264                         return;
 265 #endif
 266         }
 267
 268         if ((frame->tf_rflags & PSL_I) == 0) {
 269                 /*
 270                  * Buggy application or kernel code has disabled
 271                  * interrupts and then trapped.  Enabling interrupts
 272                  * now is wrong, but it is better than running with
 273                  * interrupts disabled until they are accidentally
 274                  * enabled later.
 275                  */
 276                 if (TRAPF_USERMODE(frame)) {
 277                         uprintf(
 278                             "pid %ld (%s): trap %d with interrupts disabled\n",
 279                             (long)curproc->p_pid, curthread->td_name, type);
 280                 } else {
 281                         switch (type) {
 282                         case T_NMI:
 283                         case T_BPTFLT:
 284                         case T_TRCTRAP:
 285                         case T_PROTFLT:
 286                         case T_SEGNPFLT:
 287                         case T_STKFLT:
 288                                 break;
 289                         default:
 290                                 printf(
 291                                     "kernel trap %d with interrupts disabled\n",
 292                                     type);
 293
 294                                 /*
 295                                  * We shouldn't enable interrupts while holding a
 296                                  * spin lock.
 297                                  */
 298                                 if (td->td_md.md_spinlock_count == 0)
 299                                         enable_intr();
 300                         }
 301                 }
 302         }
 303
 304         if (TRAPF_USERMODE(frame)) {
 305                 /* user trap */
 306
 307                 td->td_pticks = 0;
 308                 td->td_frame = frame;
 309                 addr = frame->tf_rip;
 310                 if (td->td_cowgen != p->p_cowgen)
 311                         thread_cow_update(td);
 312
 313                 switch (type) {
 314                 case T_PRIVINFLT:       /* privileged instruction fault */
 315                         signo = SIGILL;
 316                         ucode = ILL_PRVOPC;
 317                         break;
 318
 319                 case T_BPTFLT:          /* bpt instruction fault */
 320 #ifdef KDTRACE_HOOKS
 321                         if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
 322                                 return;
 323 #else
 324                         enable_intr();
 325 #endif
 326                         signo = SIGTRAP;
 327                         ucode = TRAP_BRKPT;
 328                         break;
 329
 330                 case T_TRCTRAP:         /* debug exception */
 331                         enable_intr();
 332                         signo = SIGTRAP;
 333                         ucode = TRAP_TRACE;
 334                         dr6 = rdr6();
 335                         if ((dr6 & DBREG_DR6_BS) != 0) {
 336                                 PROC_LOCK(td->td_proc);
 337                                 if ((td->td_dbgflags & TDB_STEP) != 0) {
 338                                         td->td_frame->tf_rflags &= ~PSL_T;
 339                                         td->td_dbgflags &= ~TDB_STEP;
 340                                 }
 341                                 PROC_UNLOCK(td->td_proc);
 342                         }
 343                         break;
 344
 345                 case T_ARITHTRAP:       /* arithmetic trap */
 346                         ucode = fputrap_x87();
 347                         if (ucode == -1)
 348                                 return;
 349                         signo = SIGFPE;
 350                         break;
 351
 352                 case T_PROTFLT:         /* general protection fault */
 353                         signo = SIGBUS;
 354                         ucode = BUS_OBJERR;
 355                         break;
 356                 case T_STKFLT:          /* stack fault */
 357                 case T_SEGNPFLT:        /* segment not present fault */
 358                         signo = SIGBUS;
 359                         ucode = BUS_ADRERR;
 360                         break;
 361                 case T_TSSFLT:          /* invalid TSS fault */
 362                         signo = SIGBUS;
 363                         ucode = BUS_OBJERR;
 364                         break;
 365                 case T_ALIGNFLT:
 366                         signo = SIGBUS;
 367                         ucode = BUS_ADRALN;
 368                         break;
 369                 case T_DOUBLEFLT:       /* double fault */
 370                 default:
 371                         signo = SIGBUS;
 372                         ucode = BUS_OBJERR;
 373                         break;
 374
 375                 case T_PAGEFLT:         /* page fault */
 376                         /*
 377                          * Can emulator handle this trap?
 378                          */
 379                         if (*p->p_sysent->sv_trap != NULL &&
 380                             (*p->p_sysent->sv_trap)(td) == 0)
 381                                 return;
 382
 383                         pf = trap_pfault(frame, true, &signo, &ucode);
 384                         if (pf == -1)
 385                                 return;
 386                         if (pf == 0)
 387                                 goto userret;
 388                         addr = frame->tf_addr;
 389                         break;
 390
 391                 case T_DIVIDE:          /* integer divide fault */
 392                         ucode = FPE_INTDIV;
 393                         signo = SIGFPE;
 394                         break;
 395
 396                 case T_NMI:
 397                         nmi_handle_intr(type, frame);
 398                         return;
 399
 400                 case T_OFLOW:           /* integer overflow fault */
 401                         ucode = FPE_INTOVF;
 402                         signo = SIGFPE;
 403                         break;
 404
 405                 case T_BOUND:           /* bounds check fault */
 406                         ucode = FPE_FLTSUB;
 407                         signo = SIGFPE;
 408                         break;
 409
 410                 case T_DNA:
 411                         /* transparent fault (due to context switch "late") */
 412                         KASSERT(PCB_USER_FPU(td->td_pcb),
 413                             ("kernel FPU ctx has leaked"));
 414                         fpudna();
 415                         return;
 416
 417                 case T_FPOPFLT:         /* FPU operand fetch fault */
 418                         ucode = ILL_COPROC;
 419                         signo = SIGILL;
 420                         break;
 421
 422                 case T_XMMFLT:          /* SIMD floating-point exception */
 423                         ucode = fputrap_sse();
 424                         if (ucode == -1)
 425                                 return;
 426                         signo = SIGFPE;
 427                         break;
 428 #ifdef KDTRACE_HOOKS
 429                 case T_DTRACE_RET:
 430                         (void)trap_user_dtrace(frame, &dtrace_return_probe_ptr);
 431                         return;
 432 #endif
 433                 }
 434         } else {
 435                 /* kernel trap */
 436
 437                 KASSERT(cold || td->td_ucred != NULL,
 438                     ("kernel trap doesn't have ucred"));
 439                 switch (type) {
 440                 case T_PAGEFLT:                 /* page fault */
 441                         (void)trap_pfault(frame, false, NULL, NULL);
 442                         return;
 443
 444                 case T_DNA:
 445                         if (PCB_USER_FPU(td->td_pcb))
 446                                 panic("Unregistered use of FPU in kernel");
 447                         fpudna();
 448                         return;
 449
 450                 case T_ARITHTRAP:       /* arithmetic trap */
 451                 case T_XMMFLT:          /* SIMD floating-point exception */
 452                 case T_FPOPFLT:         /* FPU operand fetch fault */
 453                         /*
 454                          * For now, supporting kernel handler
 455                          * registration for FPU traps is overkill.
 456                          */
 457                         trap_fatal(frame, 0);
 458                         return;
 459
 460                 case T_STKFLT:          /* stack fault */
 461                 case T_PROTFLT:         /* general protection fault */
 462                 case T_SEGNPFLT:        /* segment not present fault */
 463                         if (td->td_intr_nesting_level != 0)
 464                                 break;
 465
 466                         /*
 467                          * Invalid segment selectors and out of bounds
 468                          * %rip's and %rsp's can be set up in user mode.
 469                          * This causes a fault in kernel mode when the
 470                          * kernel tries to return to user mode.  We want
 471                          * to get this fault so that we can fix the
 472                          * problem here and not have to check all the
 473                          * selectors and pointers when the user changes
 474                          * them.
 475                          *
 476                          * In case of PTI, the IRETQ faulted while the
 477                          * kernel used the pti stack, and exception
 478                          * frame records %rsp value pointing to that
 479                          * stack.  If we return normally to
 480                          * doreti_iret_fault, the trapframe is
 481                          * reconstructed on pti stack, and calltrap()
 482                          * called on it as well.  Due to the very
 483                          * limited pti stack size, kernel does not
 484                          * survive for too long.  Switch to the normal
 485                          * thread stack for the trap handling.
 486                          *
 487                          * Magic '5' is the number of qwords occupied by
 488                          * the hardware trap frame.
 489                          */
 490                         if (frame->tf_rip == (long)doreti_iret) {
 491                                 KASSERT((read_rflags() & PSL_I) == 0,
 492                                     ("interrupts enabled"));
 493                                 frame->tf_rip = (long)doreti_iret_fault;
 494                                 if ((PCPU_GET(curpmap)->pm_ucr3 !=
 495                                     PMAP_NO_CR3) &&
 496                                     (frame->tf_rsp == (uintptr_t)PCPU_GET(
 497                                     pti_rsp0) - 5 * sizeof(register_t))) {
 498                                         frame->tf_rsp = PCPU_GET(rsp0) - 5 *
 499                                             sizeof(register_t);
 500                                 }
 501                                 return;
 502                         }
 503
 504                         for (i = 0; i < nitems(sfhandlers); i++) {
 505                                 if (frame->tf_rip == sfhandlers[i].faddr) {
 506                                         KASSERT((read_rflags() & PSL_I) == 0,
 507                                             ("interrupts enabled"));
 508                                         frame->tf_rip = sfhandlers[i].fhandler;
 509                                         return;
 510                                 }
 511                         }
 512
 513                         if (curpcb->pcb_onfault != NULL) {
 514                                 frame->tf_rip = (long)curpcb->pcb_onfault;
 515                                 return;
 516                         }
 517                         break;
 518
 519                 case T_TSSFLT:
 520                         /*
 521                          * PSL_NT can be set in user mode and isn't cleared
 522                          * automatically when the kernel is entered.  This
 523                          * causes a TSS fault when the kernel attempts to
 524                          * `iret' because the TSS link is uninitialized.  We
 525                          * want to get this fault so that we can fix the
 526                          * problem here and not every time the kernel is
 527                          * entered.
 528                          */
 529                         if (frame->tf_rflags & PSL_NT) {
 530                                 frame->tf_rflags &= ~PSL_NT;
 531                                 return;
 532                         }
 533                         break;
 534
 535                 case T_TRCTRAP:  /* debug exception */
 536                         /* Clear any pending debug events. */
 537                         dr6 = rdr6();
 538                         load_dr6(0);
 539
 540                         /*
 541                          * Ignore debug register exceptions due to
 542                          * accesses in the user's address space, which
 543                          * can happen under several conditions such as
 544                          * if a user sets a watchpoint on a buffer and
 545                          * then passes that buffer to a system call.
 546                          * We still want to get TRCTRAPS for addresses
 547                          * in kernel space because that is useful when
 548                          * debugging the kernel.
 549                          */
 550                         if (user_dbreg_trap(dr6))
 551                                 return;
 552
 553                         /*
 554                          * Malicious user code can configure a debug
 555                          * register watchpoint to trap on data access
 556                          * to the top of stack and then execute 'pop
 557                          * %ss; int 3'.  Due to exception deferral for
 558                          * 'pop %ss', the CPU will not interrupt 'int
 559                          * 3' to raise the DB# exception for the debug
 560                          * register but will postpone the DB# until
 561                          * execution of the first instruction of the
 562                          * BP# handler (in kernel mode).  Normally the
 563                          * previous check would ignore DB# exceptions
 564                          * for watchpoints on user addresses raised in
 565                          * kernel mode.  However, some CPU errata
 566                          * include cases where DB# exceptions do not
 567                          * properly set bits in %dr6, e.g. Haswell
 568                          * HSD23 and Skylake-X SKZ24.
 569                          *
 570                          * A deferred DB# can also be raised on the
 571                          * first instructions of system call entry
 572                          * points or single-step traps via similar use
 573                          * of 'pop %ss' or 'mov xxx, %ss'.
 574                          */
 575                         if (pti) {
 576                                 if (frame->tf_rip ==
 577                                     (uintptr_t)IDTVEC(fast_syscall_pti) ||
 578 #ifdef COMPAT_FREEBSD32
 579                                     frame->tf_rip ==
 580                                     (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
 581 #endif
 582                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
 583                                         return;
 584                         } else {
 585                                 if (frame->tf_rip ==
 586                                     (uintptr_t)IDTVEC(fast_syscall) ||
 587 #ifdef COMPAT_FREEBSD32
 588                                     frame->tf_rip ==
 589                                     (uintptr_t)IDTVEC(int0x80_syscall) ||
 590 #endif
 591                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt))
 592                                         return;
 593                         }
 594                         if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
 595                             /* Needed for AMD. */
 596                             frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
 597                                 return;
 598                         /*
 599                          * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
 600                          */
 601                 case T_BPTFLT:
 602                         /*
 603                          * If KDB is enabled, let it handle the debugger trap.
 604                          * Otherwise, debugger traps "can't happen".
 605                          */
 606 #ifdef KDB
 607                         if (kdb_trap(type, dr6, frame))
 608                                 return;
 609 #endif
 610                         break;
 611
 612                 case T_NMI:
 613                         nmi_handle_intr(type, frame);
 614                         return;
 615                 }
 616
 617                 trap_fatal(frame, 0);
 618                 return;
 619         }
 620
 621         /* Translate fault for emulators (e.g. Linux) */
 622         if (*p->p_sysent->sv_transtrap != NULL)
 623                 signo = (*p->p_sysent->sv_transtrap)(signo, type);
 624
 625         ksiginfo_init_trap(&ksi);
 626         ksi.ksi_signo = signo;
 627         ksi.ksi_code = ucode;
 628         ksi.ksi_trapno = type;
 629         ksi.ksi_addr = (void *)addr;
 630         if (uprintf_signal) {
 631                 uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
 632                     "addr 0x%lx rsp 0x%lx rip 0x%lx "
 633                     "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
 634                     p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
 635                     addr, frame->tf_rsp, frame->tf_rip,
 636                     fubyte((void *)(frame->tf_rip + 0)),
 637                     fubyte((void *)(frame->tf_rip + 1)),
 638                     fubyte((void *)(frame->tf_rip + 2)),
 639                     fubyte((void *)(frame->tf_rip + 3)),
 640                     fubyte((void *)(frame->tf_rip + 4)),
 641                     fubyte((void *)(frame->tf_rip + 5)),
 642                     fubyte((void *)(frame->tf_rip + 6)),
 643                     fubyte((void *)(frame->tf_rip + 7)));
 644         }
 645         KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
 646         trapsignal(td, &ksi);
 647
 648 userret:
 649         userret(td, frame);
 650         KASSERT(PCB_USER_FPU(td->td_pcb),
 651             ("Return from trap with kernel FPU ctx leaked"));
 652 }
 653
 654 /*
 655  * Ensure that we ignore any DTrace-induced faults. This function cannot
 656  * be instrumented, so it cannot generate such faults itself.
 657  */
 658 void
 659 trap_check(struct trapframe *frame)
 660 {
 661
 662 #ifdef KDTRACE_HOOKS
 663         if (dtrace_trap_func != NULL &&
 664             (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
 665                 return;
 666 #endif
 667         trap(frame);
 668 }
 669
 670 static bool
 671 trap_is_smap(struct trapframe *frame)
 672 {
 673
 674         /*
 675          * A page fault on a userspace address is classified as
 676          * SMAP-induced if:
 677          * - SMAP is supported;
 678          * - kernel mode accessed present data page;
 679          * - rflags.AC was cleared.
 680          * Kernel must never access user space with rflags.AC cleared
 681          * if SMAP is enabled.
 682          */
 683         return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
 684             (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
 685             PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
 686 }
 687
 688 static bool
 689 trap_is_pti(struct trapframe *frame)
 690 {
 691
 692         return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
 693             pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
 694             PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
 695             (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
 696             (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
 697 }
 698
 699 /*
 700  * Handle all details of a page fault.
 701  * Returns:
 702  * -1 if this fault was fatal, typically from kernel mode
 703  *    (cannot happen, but we need to return something).
 704  * 0  if this fault was handled by updating either the user or kernel
 705  *    page table, execution can continue.
 706  * 1  if this fault was from usermode and it was not handled, a synchronous
 707  *    signal should be delivered to the thread.  *signo returns the signal
 708  *    number, *ucode gives si_code.
 709  */
 710 static int
 711 trap_pfault(struct trapframe *frame, bool usermode, int *signo, int *ucode)
 712 {
 713         struct thread *td;
 714         struct proc *p;
 715         vm_map_t map;
 716         vm_offset_t eva;
 717         int rv;
 718         vm_prot_t ftype;
 719
 720         MPASS(!usermode || (signo != NULL && ucode != NULL));
 721
 722         td = curthread;
 723         p = td->td_proc;
 724         eva = frame->tf_addr;
 725
 726         if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
 727                 /*
 728                  * Due to both processor errata and lazy TLB invalidation when
 729                  * access restrictions are removed from virtual pages, memory
 730                  * accesses that are allowed by the physical mapping layer may
 731                  * nonetheless cause one spurious page fault per virtual page.
 732                  * When the thread is executing a "no faulting" section that
 733                  * is bracketed by vm_fault_{disable,enable}_pagefaults(),
 734                  * every page fault is treated as a spurious page fault,
 735                  * unless it accesses the same virtual address as the most
 736                  * recent page fault within the same "no faulting" section.
 737                  */
 738                 if (td->td_md.md_spurflt_addr != eva ||
 739                     (td->td_pflags & TDP_RESETSPUR) != 0) {
 740                         /*
 741                          * Do nothing to the TLB.  A stale TLB entry is
 742                          * flushed automatically by a page fault.
 743                          */
 744                         td->td_md.md_spurflt_addr = eva;
 745                         td->td_pflags &= ~TDP_RESETSPUR;
 746                         return (0);
 747                 }
 748         } else {
 749                 /*
 750                  * If we get a page fault while in a critical section, then
 751                  * it is most likely a fatal kernel page fault.  The kernel
 752                  * is already going to panic trying to get a sleep lock to
 753                  * do the VM lookup, so just consider it a fatal trap so the
 754                  * kernel can print out a useful trap message and even get
 755                  * to the debugger.
 756                  *
 757                  * If we get a page fault while holding a non-sleepable
 758                  * lock, then it is most likely a fatal kernel page fault.
 759                  * If WITNESS is enabled, then it's going to whine about
 760                  * bogus LORs with various VM locks, so just skip to the
 761                  * fatal trap handling directly.
 762                  */
 763                 if (td->td_critnest != 0 ||
 764                     WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
 765                     "Kernel page fault") != 0) {
 766                         trap_fatal(frame, eva);
 767                         return (-1);
 768                 }
 769         }
 770         if (eva >= VM_MIN_KERNEL_ADDRESS) {
 771                 /*
 772                  * Don't allow user-mode faults in kernel address space.
 773                  */
 774                 if (usermode) {
 775                         *signo = SIGSEGV;
 776                         *ucode = SEGV_MAPERR;
 777                         return (1);
 778                 }
 779
 780                 map = kernel_map;
 781         } else {
 782                 map = &p->p_vmspace->vm_map;
 783
 784                 /*
 785                  * When accessing a usermode address, kernel must be
 786                  * ready to accept the page fault, and provide a
 787                  * handling routine.  Since accessing the address
 788                  * without the handler is a bug, do not try to handle
 789                  * it normally, and panic immediately.
 790                  *
 791                  * If SMAP is enabled, filter SMAP faults also,
 792                  * because illegal access might occur to the mapped
 793                  * user address, causing infinite loop.
 794                  */
 795                 if (!usermode && (td->td_intr_nesting_level != 0 ||
 796                     trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
 797                         trap_fatal(frame, eva);
 798                         return (-1);
 799                 }
 800         }
 801
 802         /*
 803          * If the trap was caused by errant bits in the PTE then panic.
 804          */
 805         if (frame->tf_err & PGEX_RSV) {
 806                 trap_fatal(frame, eva);
 807                 return (-1);
 808         }
 809
 810         /*
 811          * User-mode protection key violation (PKU).  May happen
 812          * either from usermode or from kernel if copyin accessed
 813          * key-protected mapping.
 814          */
 815         if ((frame->tf_err & PGEX_PK) != 0) {
 816                 if (eva > VM_MAXUSER_ADDRESS) {
 817                         trap_fatal(frame, eva);
 818                         return (-1);
 819                 }
 820                 if (usermode) {
 821                         *signo = SIGSEGV;
 822                         *ucode = SEGV_PKUERR;
 823                         return (1);
 824                 }
 825                 goto after_vmfault;
 826         }
 827
 828         /*
 829          * If nx protection of the usermode portion of kernel page
 830          * tables caused trap, panic.
 831          */
 832         if (usermode && trap_is_pti(frame))
 833                 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
 834                     p->p_comm, frame->tf_err);
 835
 836         /*
 837          * PGEX_I is defined only if the execute disable bit capability is
 838          * supported and enabled.
 839          */
 840         if (frame->tf_err & PGEX_W)
 841                 ftype = VM_PROT_WRITE;
 842         else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
 843                 ftype = VM_PROT_EXECUTE;
 844         else
 845                 ftype = VM_PROT_READ;
 846
 847         /* Fault in the page. */
 848         rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
 849         if (rv == KERN_SUCCESS) {
 850 #ifdef HWPMC_HOOKS
 851                 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
 852                         PMC_SOFT_CALL_TF( , , page_fault, all, frame);
 853                         if (ftype == VM_PROT_READ)
 854                                 PMC_SOFT_CALL_TF( , , page_fault, read,
 855                                     frame);
 856                         else
 857                                 PMC_SOFT_CALL_TF( , , page_fault, write,
 858                                     frame);
 859                 }
 860 #endif
 861                 return (0);
 862         }
 863
 864         if (usermode)
 865                 return (1);
 866 after_vmfault:
 867         if (td->td_intr_nesting_level == 0 &&
 868             curpcb->pcb_onfault != NULL) {
 869                 frame->tf_rip = (long)curpcb->pcb_onfault;
 870                 return (0);
 871         }
 872         trap_fatal(frame, eva);
 873         return (-1);
 874 }
 875
 876 static void
 877 trap_fatal(struct trapframe *frame, vm_offset_t eva)
 878 {
 879         int code, ss;
 880         u_int type;
 881         struct soft_segment_descriptor softseg;
 882         struct user_segment_descriptor *gdt;
 883 #ifdef KDB
 884         bool handled;
 885 #endif
 886
 887         code = frame->tf_err;
 888         type = frame->tf_trapno;
 889         gdt = *PCPU_PTR(gdt);
 890         sdtossd(&gdt[IDXSEL(frame->tf_cs & 0xffff)], &softseg);
 891
 892         printf("\n\nFatal trap %d: %s while in %s mode\n", type,
 893             type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN,
 894             TRAPF_USERMODE(frame) ? "user" : "kernel");
 895 #ifdef SMP
 896         /* two separate prints in case of a trap on an unmapped page */
 897         printf("cpuid = %d; ", PCPU_GET(cpuid));
 898         printf("apic id = %02x\n", PCPU_GET(apic_id));
 899 #endif
 900         if (type == T_PAGEFLT) {
 901                 printf("fault virtual address   = 0x%lx\n", eva);
 902                 printf("fault code              = %s %s %s%s%s, %s\n",
 903                         code & PGEX_U ? "user" : "supervisor",
 904                         code & PGEX_W ? "write" : "read",
 905                         code & PGEX_I ? "instruction" : "data",
 906                         code & PGEX_PK ? " prot key" : "",
 907                         code & PGEX_SGX ? " SGX" : "",
 908                         code & PGEX_RSV ? "reserved bits in PTE" :
 909                         code & PGEX_P ? "protection violation" : "page not present");
 910         }
 911         printf("instruction pointer     = 0x%lx:0x%lx\n",
 912                frame->tf_cs & 0xffff, frame->tf_rip);
 913         ss = frame->tf_ss & 0xffff;
 914         printf("stack pointer           = 0x%x:0x%lx\n", ss, frame->tf_rsp);
 915         printf("frame pointer           = 0x%x:0x%lx\n", ss, frame->tf_rbp);
 916         printf("code segment            = base 0x%lx, limit 0x%lx, type 0x%x\n",
 917                softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
 918         printf("                        = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
 919                softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
 920                softseg.ssd_gran);
 921         printf("processor eflags        = ");
 922         if (frame->tf_rflags & PSL_T)
 923                 printf("trace trap, ");
 924         if (frame->tf_rflags & PSL_I)
 925                 printf("interrupt enabled, ");
 926         if (frame->tf_rflags & PSL_NT)
 927                 printf("nested task, ");
 928         if (frame->tf_rflags & PSL_RF)
 929                 printf("resume, ");
 930         printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
 931         printf("current process         = %d (%s)\n",
 932             curproc->p_pid, curthread->td_name);
 933
 934 #ifdef KDB
 935         if (debugger_on_trap) {
 936                 kdb_why = KDB_WHY_TRAP;
 937                 handled = kdb_trap(type, 0, frame);
 938                 kdb_why = KDB_WHY_UNSET;
 939                 if (handled)
 940                         return;
 941         }
 942 #endif
 943         printf("trap number             = %d\n", type);
 944         panic("%s", type < nitems(trap_msg) ? trap_msg[type] :
 945             "unknown/reserved trap");
 946 }
 947
 948 #ifdef KDTRACE_HOOKS
 949 /*
 950  * Invoke a userspace DTrace hook.  The hook pointer is cleared when no
 951  * userspace probes are enabled, so we must synchronize with DTrace to ensure
 952  * that a trapping thread is able to call the hook before it is cleared.
 953  */
 954 static bool
 955 trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
 956 {
 957         int (*hook)(struct trapframe *);
 958
 959         hook = atomic_load_ptr(hookp);
 960         enable_intr();
 961         if (hook != NULL)
 962                 return ((hook)(frame) == 0);
 963         return (false);
 964 }
 965 #endif
 966
 967 /*
 968  * Double fault handler. Called when a fault occurs while writing
 969  * a frame for a trap/exception onto the stack. This usually occurs
 970  * when the stack overflows (such is the case with infinite recursion,
 971  * for example).
 972  */
 973 void
 974 dblfault_handler(struct trapframe *frame)
 975 {
 976 #ifdef KDTRACE_HOOKS
 977         if (dtrace_doubletrap_func != NULL)
 978                 (*dtrace_doubletrap_func)();
 979 #endif
 980         printf("\nFatal double fault\n"
 981             "rip %#lx rsp %#lx rbp %#lx\n"
 982             "rax %#lx rdx %#lx rbx %#lx\n"
 983             "rcx %#lx rsi %#lx rdi %#lx\n"
 984             "r8 %#lx r9 %#lx r10 %#lx\n"
 985             "r11 %#lx r12 %#lx r13 %#lx\n"
 986             "r14 %#lx r15 %#lx rflags %#lx\n"
 987             "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
 988             "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
 989             frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
 990             frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
 991             frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
 992             frame->tf_r8, frame->tf_r9, frame->tf_r10,
 993             frame->tf_r11, frame->tf_r12, frame->tf_r13,
 994             frame->tf_r14, frame->tf_r15, frame->tf_rflags,
 995             frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
 996             frame->tf_fs, frame->tf_gs,
 997             rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
 998 #ifdef SMP
 999         /* two separate prints in case of a trap on an unmapped page */
1000         printf("cpuid = %d; ", PCPU_GET(cpuid));
1001         printf("apic id = %02x\n", PCPU_GET(apic_id));
1002 #endif
1003         panic("double fault");
1004 }
1005
1006 static int __noinline
1007 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
1008 {
1009         struct proc *p;
1010         struct trapframe *frame;
1011         register_t *argp;
1012         caddr_t params;
1013         int reg, regcnt, error;
1014
1015         p = td->td_proc;
1016         frame = td->td_frame;
1017         reg = 0;
1018         regcnt = NARGREGS;
1019
1020         if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
1021                 sa->code = frame->tf_rdi;
1022                 reg++;
1023                 regcnt--;
1024         }
1025
1026         if (sa->code >= p->p_sysent->sv_size)
1027                 sa->callp = &p->p_sysent->sv_table[0];
1028         else
1029                 sa->callp = &p->p_sysent->sv_table[sa->code];
1030
1031         KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1032             ("Too many syscall arguments!"));
1033         argp = &frame->tf_rdi;
1034         argp += reg;
1035         memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
1036         if (sa->callp->sy_narg > regcnt) {
1037                 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
1038                 error = copyin(params, &sa->args[regcnt],
1039                     (sa->callp->sy_narg - regcnt) * sizeof(sa->args[0]));
1040                 if (__predict_false(error != 0))
1041                         return (error);
1042         }
1043
1044         td->td_retval[0] = 0;
1045         td->td_retval[1] = frame->tf_rdx;
1046
1047         return (0);
1048 }
1049
1050 int
1051 cpu_fetch_syscall_args(struct thread *td)
1052 {
1053         struct proc *p;
1054         struct trapframe *frame;
1055         struct syscall_args *sa;
1056
1057         p = td->td_proc;
1058         frame = td->td_frame;
1059         sa = &td->td_sa;
1060
1061         sa->code = frame->tf_rax;
1062
1063         if (__predict_false(sa->code == SYS_syscall ||
1064             sa->code == SYS___syscall ||
1065             sa->code >= p->p_sysent->sv_size))
1066                 return (cpu_fetch_syscall_args_fallback(td, sa));
1067
1068         sa->callp = &p->p_sysent->sv_table[sa->code];
1069         KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1070             ("Too many syscall arguments!"));
1071
1072         if (__predict_false(sa->callp->sy_narg > NARGREGS))
1073                 return (cpu_fetch_syscall_args_fallback(td, sa));
1074
1075         memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
1076
1077         td->td_retval[0] = 0;
1078         td->td_retval[1] = frame->tf_rdx;
1079
1080         return (0);
1081 }
1082
1083 #include "../../kern/subr_syscall.c"
1084
1085 static void (*syscall_ret_l1d_flush)(void);
1086 int syscall_ret_l1d_flush_mode;
1087
1088 static void
1089 flush_l1d_hw(void)
1090 {
1091
1092         wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
1093 }
1094
1095 static void __noinline
1096 amd64_syscall_ret_flush_l1d_check(int error)
1097 {
1098         void (*p)(void);
1099
1100         if (error != EEXIST && error != EAGAIN && error != EXDEV &&
1101             error != ENOENT && error != ENOTCONN && error != EINPROGRESS) {
1102                 p = atomic_load_ptr(&syscall_ret_l1d_flush);
1103                 if (p != NULL)
1104                         p();
1105         }
1106 }
1107
1108 static void __inline
1109 amd64_syscall_ret_flush_l1d_check_inline(int error)
1110 {
1111
1112         if (__predict_false(error != 0))
1113                 amd64_syscall_ret_flush_l1d_check(error);
1114 }
1115
1116 void
1117 amd64_syscall_ret_flush_l1d(int error)
1118 {
1119
1120         amd64_syscall_ret_flush_l1d_check_inline(error);
1121 }
1122
1123 void
1124 amd64_syscall_ret_flush_l1d_recalc(void)
1125 {
1126         bool l1d_hw;
1127
1128         l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
1129 again:
1130         switch (syscall_ret_l1d_flush_mode) {
1131         case 0:
1132                 syscall_ret_l1d_flush = NULL;
1133                 break;
1134         case 1:
1135                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
1136                     flush_l1d_sw_abi;
1137                 break;
1138         case 2:
1139                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
1140                 break;
1141         case 3:
1142                 syscall_ret_l1d_flush = flush_l1d_sw_abi;
1143                 break;
1144         default:
1145                 syscall_ret_l1d_flush_mode = 1;
1146                 goto again;
1147         }
1148 }
1149
1150 static int
1151 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
1152 {
1153         int error, val;
1154
1155         val = syscall_ret_l1d_flush_mode;
1156         error = sysctl_handle_int(oidp, &val, 0, req);
1157         if (error != 0 || req->newptr == NULL)
1158                 return (error);
1159         syscall_ret_l1d_flush_mode = val;
1160         amd64_syscall_ret_flush_l1d_recalc();
1161         return (0);
1162 }
1163 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
1164     CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
1165     machdep_syscall_ret_flush_l1d, "I",
1166     "Flush L1D on syscall return with error (0 - off, 1 - on, "
1167     "2 - use hw only, 3 - use sw only)");
1168
1169 /*
1170  * System call handler for native binaries.  The trap frame is already
1171  * set up by the assembler trampoline and a pointer to it is saved in
1172  * td_frame.
1173  */
1174 void
1175 amd64_syscall(struct thread *td, int traced)
1176 {
1177         ksiginfo_t ksi;
1178
1179 #ifdef DIAGNOSTIC
1180         if (!TRAPF_USERMODE(td->td_frame)) {
1181                 panic("syscall");
1182                 /* NOT REACHED */
1183         }
1184 #endif
1185         syscallenter(td);
1186
1187         /*
1188          * Traced syscall.
1189          */
1190         if (__predict_false(traced)) {
1191                 td->td_frame->tf_rflags &= ~PSL_T;
1192                 ksiginfo_init_trap(&ksi);
1193                 ksi.ksi_signo = SIGTRAP;
1194                 ksi.ksi_code = TRAP_TRACE;
1195                 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
1196                 trapsignal(td, &ksi);
1197         }
1198
1199         KASSERT(PCB_USER_FPU(td->td_pcb),
1200             ("System call %s returning with kernel FPU ctx leaked",
1201              syscallname(td->td_proc, td->td_sa.code)));
1202         KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
1203             ("System call %s returning with mangled pcb_save",
1204              syscallname(td->td_proc, td->td_sa.code)));
1205         KASSERT(pmap_not_in_di(),
1206             ("System call %s returning with leaked invl_gen %lu",
1207             syscallname(td->td_proc, td->td_sa.code),
1208             td->td_md.md_invl_gen.gen));
1209
1210         syscallret(td);
1211
1212         /*
1213          * If the user-supplied value of %rip is not a canonical
1214          * address, then some CPUs will trigger a ring 0 #GP during
1215          * the sysret instruction.  However, the fault handler would
1216          * execute in ring 0 with the user's %gs and %rsp which would
1217          * not be safe.  Instead, use the full return path which
1218          * catches the problem safely.
1219          */
1220         if (__predict_false(td->td_frame->tf_rip >= (la57 ?
1221             VM_MAXUSER_ADDRESS_LA57 : VM_MAXUSER_ADDRESS_LA48)))
1222                 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
1223
1224         amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
1225 }