sys/amd64/amd64/trap.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (C) 1994, David Greenman
   5  * Copyright (c) 1990, 1993
   6  *      The Regents of the University of California.  All rights reserved.
   7  *
   8  * This code is derived from software contributed to Berkeley by
   9  * the University of Utah, and William Jolitz.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by the University of
  22  *      California, Berkeley and its contributors.
  23  * 4. Neither the name of the University nor the names of its contributors
  24  *    may be used to endorse or promote products derived from this software
  25  *    without specific prior written permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  37  * SUCH DAMAGE.
  38  *
  39  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
  40  */
  41
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 /*
  46  * AMD64 Trap and System call handling
  47  */
  48
  49 #include "opt_clock.h"
  50 #include "opt_compat.h"
  51 #include "opt_cpu.h"
  52 #include "opt_hwpmc_hooks.h"
  53 #include "opt_isa.h"
  54 #include "opt_kdb.h"
  55 #include "opt_stack.h"
  56
  57 #include <sys/param.h>
  58 #include <sys/bus.h>
  59 #include <sys/systm.h>
  60 #include <sys/proc.h>
  61 #include <sys/pioctl.h>
  62 #include <sys/ptrace.h>
  63 #include <sys/kdb.h>
  64 #include <sys/kernel.h>
  65 #include <sys/ktr.h>
  66 #include <sys/lock.h>
  67 #include <sys/mutex.h>
  68 #include <sys/resourcevar.h>
  69 #include <sys/signalvar.h>
  70 #include <sys/syscall.h>
  71 #include <sys/sysctl.h>
  72 #include <sys/sysent.h>
  73 #include <sys/uio.h>
  74 #include <sys/vmmeter.h>
  75 #ifdef HWPMC_HOOKS
  76 #include <sys/pmckern.h>
  77 PMC_SOFT_DEFINE( , , page_fault, all);
  78 PMC_SOFT_DEFINE( , , page_fault, read);
  79 PMC_SOFT_DEFINE( , , page_fault, write);
  80 #endif
  81
  82 #include <vm/vm.h>
  83 #include <vm/vm_param.h>
  84 #include <vm/pmap.h>
  85 #include <vm/vm_kern.h>
  86 #include <vm/vm_map.h>
  87 #include <vm/vm_page.h>
  88 #include <vm/vm_extern.h>
  89
  90 #include <machine/cpu.h>
  91 #include <machine/intr_machdep.h>
  92 #include <x86/mca.h>
  93 #include <machine/md_var.h>
  94 #include <machine/pcb.h>
  95 #ifdef SMP
  96 #include <machine/smp.h>
  97 #endif
  98 #include <machine/stack.h>
  99 #include <machine/trap.h>
 100 #include <machine/tss.h>
 101
 102 #ifdef KDTRACE_HOOKS
 103 #include <sys/dtrace_bsd.h>
 104 #endif
 105
 106 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
 107     IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
 108     IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
 109
 110 void __noinline trap(struct trapframe *frame);
 111 void trap_check(struct trapframe *frame);
 112 void dblfault_handler(struct trapframe *frame);
 113
 114 static int trap_pfault(struct trapframe *, bool, int *, int *);
 115 static void trap_fatal(struct trapframe *, vm_offset_t);
 116 #ifdef KDTRACE_HOOKS
 117 static bool trap_user_dtrace(struct trapframe *,
 118     int (**hook)(struct trapframe *));
 119 #endif
 120
 121 static const char UNKNOWN[] = "unknown";
 122 static const char *const trap_msg[] = {
 123         [0] =                   UNKNOWN,                        /* unused */
 124         [T_PRIVINFLT] =         "privileged instruction fault",
 125         [2] =                   UNKNOWN,                        /* unused */
 126         [T_BPTFLT] =            "breakpoint instruction fault",
 127         [4] =                   UNKNOWN,                        /* unused */
 128         [5] =                   UNKNOWN,                        /* unused */
 129         [T_ARITHTRAP] =         "arithmetic trap",
 130         [7] =                   UNKNOWN,                        /* unused */
 131         [8] =                   UNKNOWN,                        /* unused */
 132         [T_PROTFLT] =           "general protection fault",
 133         [T_TRCTRAP] =           "debug exception",
 134         [11] =                  UNKNOWN,                        /* unused */
 135         [T_PAGEFLT] =           "page fault",
 136         [13] =                  UNKNOWN,                        /* unused */
 137         [T_ALIGNFLT] =          "alignment fault",
 138         [15] =                  UNKNOWN,                        /* unused */
 139         [16] =                  UNKNOWN,                        /* unused */
 140         [17] =                  UNKNOWN,                        /* unused */
 141         [T_DIVIDE] =            "integer divide fault",
 142         [T_NMI] =               "non-maskable interrupt trap",
 143         [T_OFLOW] =             "overflow trap",
 144         [T_BOUND] =             "FPU bounds check fault",
 145         [T_DNA] =               "FPU device not available",
 146         [T_DOUBLEFLT] =         "double fault",
 147         [T_FPOPFLT] =           "FPU operand fetch fault",
 148         [T_TSSFLT] =            "invalid TSS fault",
 149         [T_SEGNPFLT] =          "segment not present fault",
 150         [T_STKFLT] =            "stack fault",
 151         [T_MCHK] =              "machine check trap",
 152         [T_XMMFLT] =            "SIMD floating-point exception",
 153         [T_RESERVED] =          "reserved (unknown) fault",
 154         [31] =                  UNKNOWN,                        /* reserved */
 155         [T_DTRACE_RET] =        "DTrace pid return trap",
 156 };
 157
 158 static int uprintf_signal;
 159 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
 160     &uprintf_signal, 0,
 161     "Print debugging information on trap signal to ctty");
 162
 163 /*
 164  * Control L1D flush on return from NMI.
 165  *
 166  * Tunable  can be set to the following values:
 167  * 0 - only enable flush on return from NMI if required by vmm.ko (default)
 168  * >1 - always flush on return from NMI.
 169  *
 170  * Post-boot, the sysctl indicates if flushing is currently enabled.
 171  */
 172 int nmi_flush_l1d_sw;
 173 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
 174     &nmi_flush_l1d_sw, 0,
 175     "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
 176
 177 /*
 178  * Exception, fault, and trap interface to the FreeBSD kernel.
 179  * This common code is called from assembly language IDT gate entry
 180  * routines that prepare a suitable stack frame, and restore this
 181  * frame after the exception has been processed.
 182  */
 183
 184 void
 185 trap(struct trapframe *frame)
 186 {
 187         ksiginfo_t ksi;
 188         struct thread *td;
 189         struct proc *p;
 190         register_t addr, dr6;
 191         int pf, signo, ucode;
 192         u_int type;
 193
 194         td = curthread;
 195         p = td->td_proc;
 196         dr6 = 0;
 197
 198         VM_CNT_INC(v_trap);
 199         type = frame->tf_trapno;
 200
 201 #ifdef SMP
 202         /* Handler for NMI IPIs used for stopping CPUs. */
 203         if (type == T_NMI && ipi_nmi_handler() == 0)
 204                 return;
 205 #endif
 206
 207 #ifdef KDB
 208         if (kdb_active) {
 209                 kdb_reenter();
 210                 return;
 211         }
 212 #endif
 213
 214         if (type == T_RESERVED) {
 215                 trap_fatal(frame, 0);
 216                 return;
 217         }
 218
 219         if (type == T_NMI) {
 220 #ifdef HWPMC_HOOKS
 221                 /*
 222                  * CPU PMCs interrupt using an NMI.  If the PMC module is
 223                  * active, pass the 'rip' value to the PMC module's interrupt
 224                  * handler.  A non-zero return value from the handler means that
 225                  * the NMI was consumed by it and we can return immediately.
 226                  */
 227                 if (pmc_intr != NULL &&
 228                     (*pmc_intr)(frame) != 0)
 229                         return;
 230 #endif
 231
 232 #ifdef STACK
 233                 if (stack_nmi_handler(frame) != 0)
 234                         return;
 235 #endif
 236         }
 237
 238         if ((frame->tf_rflags & PSL_I) == 0) {
 239                 /*
 240                  * Buggy application or kernel code has disabled
 241                  * interrupts and then trapped.  Enabling interrupts
 242                  * now is wrong, but it is better than running with
 243                  * interrupts disabled until they are accidentally
 244                  * enabled later.
 245                  */
 246                 if (TRAPF_USERMODE(frame))
 247                         uprintf(
 248                             "pid %ld (%s): trap %d with interrupts disabled\n",
 249                             (long)curproc->p_pid, curthread->td_name, type);
 250                 else if (type != T_NMI && type != T_BPTFLT &&
 251                     type != T_TRCTRAP) {
 252                         /*
 253                          * XXX not quite right, since this may be for a
 254                          * multiple fault in user mode.
 255                          */
 256                         printf("kernel trap %d with interrupts disabled\n",
 257                             type);
 258
 259                         /*
 260                          * We shouldn't enable interrupts while holding a
 261                          * spin lock.
 262                          */
 263                         if (td->td_md.md_spinlock_count == 0)
 264                                 enable_intr();
 265                 }
 266         }
 267
 268         if (TRAPF_USERMODE(frame)) {
 269                 /* user trap */
 270
 271                 td->td_pticks = 0;
 272                 td->td_frame = frame;
 273                 addr = frame->tf_rip;
 274                 if (td->td_cowgen != p->p_cowgen)
 275                         thread_cow_update(td);
 276
 277                 switch (type) {
 278                 case T_PRIVINFLT:       /* privileged instruction fault */
 279                         signo = SIGILL;
 280                         ucode = ILL_PRVOPC;
 281                         break;
 282
 283                 case T_BPTFLT:          /* bpt instruction fault */
 284 #ifdef KDTRACE_HOOKS
 285                         if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
 286                                 return;
 287 #else
 288                         enable_intr();
 289 #endif
 290                         signo = SIGTRAP;
 291                         ucode = TRAP_BRKPT;
 292                         break;
 293
 294                 case T_TRCTRAP:         /* debug exception */
 295                         enable_intr();
 296                         signo = SIGTRAP;
 297                         ucode = TRAP_TRACE;
 298                         dr6 = rdr6();
 299                         if ((dr6 & DBREG_DR6_BS) != 0) {
 300                                 PROC_LOCK(td->td_proc);
 301                                 if ((td->td_dbgflags & TDB_STEP) != 0) {
 302                                         td->td_frame->tf_rflags &= ~PSL_T;
 303                                         td->td_dbgflags &= ~TDB_STEP;
 304                                 }
 305                                 PROC_UNLOCK(td->td_proc);
 306                         }
 307                         break;
 308
 309                 case T_ARITHTRAP:       /* arithmetic trap */
 310                         ucode = fputrap_x87();
 311                         if (ucode == -1)
 312                                 return;
 313                         signo = SIGFPE;
 314                         break;
 315
 316                 case T_PROTFLT:         /* general protection fault */
 317                         signo = SIGBUS;
 318                         ucode = BUS_OBJERR;
 319                         break;
 320                 case T_STKFLT:          /* stack fault */
 321                 case T_SEGNPFLT:        /* segment not present fault */
 322                         signo = SIGBUS;
 323                         ucode = BUS_ADRERR;
 324                         break;
 325                 case T_TSSFLT:          /* invalid TSS fault */
 326                         signo = SIGBUS;
 327                         ucode = BUS_OBJERR;
 328                         break;
 329                 case T_ALIGNFLT:
 330                         signo = SIGBUS;
 331                         ucode = BUS_ADRALN;
 332                         break;
 333                 case T_DOUBLEFLT:       /* double fault */
 334                 default:
 335                         signo = SIGBUS;
 336                         ucode = BUS_OBJERR;
 337                         break;
 338
 339                 case T_PAGEFLT:         /* page fault */
 340                         /*
 341                          * Can emulator handle this trap?
 342                          */
 343                         if (*p->p_sysent->sv_trap != NULL &&
 344                             (*p->p_sysent->sv_trap)(td) == 0)
 345                                 return;
 346
 347                         pf = trap_pfault(frame, true, &signo, &ucode);
 348                         if (pf == -1)
 349                                 return;
 350                         if (pf == 0)
 351                                 goto userret;
 352                         addr = frame->tf_addr;
 353                         break;
 354
 355                 case T_DIVIDE:          /* integer divide fault */
 356                         ucode = FPE_INTDIV;
 357                         signo = SIGFPE;
 358                         break;
 359
 360                 case T_NMI:
 361                         nmi_handle_intr(type, frame);
 362                         return;
 363
 364                 case T_OFLOW:           /* integer overflow fault */
 365                         ucode = FPE_INTOVF;
 366                         signo = SIGFPE;
 367                         break;
 368
 369                 case T_BOUND:           /* bounds check fault */
 370                         ucode = FPE_FLTSUB;
 371                         signo = SIGFPE;
 372                         break;
 373
 374                 case T_DNA:
 375                         /* transparent fault (due to context switch "late") */
 376                         KASSERT(PCB_USER_FPU(td->td_pcb),
 377                             ("kernel FPU ctx has leaked"));
 378                         fpudna();
 379                         return;
 380
 381                 case T_FPOPFLT:         /* FPU operand fetch fault */
 382                         ucode = ILL_COPROC;
 383                         signo = SIGILL;
 384                         break;
 385
 386                 case T_XMMFLT:          /* SIMD floating-point exception */
 387                         ucode = fputrap_sse();
 388                         if (ucode == -1)
 389                                 return;
 390                         signo = SIGFPE;
 391                         break;
 392 #ifdef KDTRACE_HOOKS
 393                 case T_DTRACE_RET:
 394                         (void)trap_user_dtrace(frame, &dtrace_return_probe_ptr);
 395                         return;
 396 #endif
 397                 }
 398         } else {
 399                 /* kernel trap */
 400
 401                 KASSERT(cold || td->td_ucred != NULL,
 402                     ("kernel trap doesn't have ucred"));
 403                 switch (type) {
 404                 case T_PAGEFLT:                 /* page fault */
 405                         (void)trap_pfault(frame, false, NULL, NULL);
 406                         return;
 407
 408                 case T_DNA:
 409                         if (PCB_USER_FPU(td->td_pcb))
 410                                 panic("Unregistered use of FPU in kernel");
 411                         fpudna();
 412                         return;
 413
 414                 case T_ARITHTRAP:       /* arithmetic trap */
 415                 case T_XMMFLT:          /* SIMD floating-point exception */
 416                 case T_FPOPFLT:         /* FPU operand fetch fault */
 417                         /*
 418                          * For now, supporting kernel handler
 419                          * registration for FPU traps is overkill.
 420                          */
 421                         trap_fatal(frame, 0);
 422                         return;
 423
 424                 case T_STKFLT:          /* stack fault */
 425                 case T_PROTFLT:         /* general protection fault */
 426                 case T_SEGNPFLT:        /* segment not present fault */
 427                         if (td->td_intr_nesting_level != 0)
 428                                 break;
 429
 430                         /*
 431                          * Invalid segment selectors and out of bounds
 432                          * %rip's and %rsp's can be set up in user mode.
 433                          * This causes a fault in kernel mode when the
 434                          * kernel tries to return to user mode.  We want
 435                          * to get this fault so that we can fix the
 436                          * problem here and not have to check all the
 437                          * selectors and pointers when the user changes
 438                          * them.
 439                          *
 440                          * In case of PTI, the IRETQ faulted while the
 441                          * kernel used the pti stack, and exception
 442                          * frame records %rsp value pointing to that
 443                          * stack.  If we return normally to
 444                          * doreti_iret_fault, the trapframe is
 445                          * reconstructed on pti stack, and calltrap()
 446                          * called on it as well.  Due to the very
 447                          * limited pti stack size, kernel does not
 448                          * survive for too long.  Switch to the normal
 449                          * thread stack for the trap handling.
 450                          *
 451                          * Magic '5' is the number of qwords occupied by
 452                          * the hardware trap frame.
 453                          */
 454                         if (frame->tf_rip == (long)doreti_iret) {
 455                                 frame->tf_rip = (long)doreti_iret_fault;
 456                                 if ((PCPU_GET(curpmap)->pm_ucr3 !=
 457                                     PMAP_NO_CR3) &&
 458                                     (frame->tf_rsp == (uintptr_t)PCPU_GET(
 459                                     pti_rsp0) - 5 * sizeof(register_t))) {
 460                                         frame->tf_rsp = PCPU_GET(rsp0) - 5 *
 461                                             sizeof(register_t);
 462                                 }
 463                                 return;
 464                         }
 465                         if (frame->tf_rip == (long)ld_ds) {
 466                                 frame->tf_rip = (long)ds_load_fault;
 467                                 return;
 468                         }
 469                         if (frame->tf_rip == (long)ld_es) {
 470                                 frame->tf_rip = (long)es_load_fault;
 471                                 return;
 472                         }
 473                         if (frame->tf_rip == (long)ld_fs) {
 474                                 frame->tf_rip = (long)fs_load_fault;
 475                                 return;
 476                         }
 477                         if (frame->tf_rip == (long)ld_gs) {
 478                                 frame->tf_rip = (long)gs_load_fault;
 479                                 return;
 480                         }
 481                         if (frame->tf_rip == (long)ld_gsbase) {
 482                                 frame->tf_rip = (long)gsbase_load_fault;
 483                                 return;
 484                         }
 485                         if (frame->tf_rip == (long)ld_fsbase) {
 486                                 frame->tf_rip = (long)fsbase_load_fault;
 487                                 return;
 488                         }
 489                         if (curpcb->pcb_onfault != NULL) {
 490                                 frame->tf_rip = (long)curpcb->pcb_onfault;
 491                                 return;
 492                         }
 493                         break;
 494
 495                 case T_TSSFLT:
 496                         /*
 497                          * PSL_NT can be set in user mode and isn't cleared
 498                          * automatically when the kernel is entered.  This
 499                          * causes a TSS fault when the kernel attempts to
 500                          * `iret' because the TSS link is uninitialized.  We
 501                          * want to get this fault so that we can fix the
 502                          * problem here and not every time the kernel is
 503                          * entered.
 504                          */
 505                         if (frame->tf_rflags & PSL_NT) {
 506                                 frame->tf_rflags &= ~PSL_NT;
 507                                 return;
 508                         }
 509                         break;
 510
 511                 case T_TRCTRAP:  /* debug exception */
 512                         /* Clear any pending debug events. */
 513                         dr6 = rdr6();
 514                         load_dr6(0);
 515
 516                         /*
 517                          * Ignore debug register exceptions due to
 518                          * accesses in the user's address space, which
 519                          * can happen under several conditions such as
 520                          * if a user sets a watchpoint on a buffer and
 521                          * then passes that buffer to a system call.
 522                          * We still want to get TRCTRAPS for addresses
 523                          * in kernel space because that is useful when
 524                          * debugging the kernel.
 525                          */
 526                         if (user_dbreg_trap(dr6))
 527                                 return;
 528
 529                         /*
 530                          * Malicious user code can configure a debug
 531                          * register watchpoint to trap on data access
 532                          * to the top of stack and then execute 'pop
 533                          * %ss; int 3'.  Due to exception deferral for
 534                          * 'pop %ss', the CPU will not interrupt 'int
 535                          * 3' to raise the DB# exception for the debug
 536                          * register but will postpone the DB# until
 537                          * execution of the first instruction of the
 538                          * BP# handler (in kernel mode).  Normally the
 539                          * previous check would ignore DB# exceptions
 540                          * for watchpoints on user addresses raised in
 541                          * kernel mode.  However, some CPU errata
 542                          * include cases where DB# exceptions do not
 543                          * properly set bits in %dr6, e.g. Haswell
 544                          * HSD23 and Skylake-X SKZ24.
 545                          *
 546                          * A deferred DB# can also be raised on the
 547                          * first instructions of system call entry
 548                          * points or single-step traps via similar use
 549                          * of 'pop %ss' or 'mov xxx, %ss'.
 550                          */
 551                         if (pti) {
 552                                 if (frame->tf_rip ==
 553                                     (uintptr_t)IDTVEC(fast_syscall_pti) ||
 554 #ifdef COMPAT_FREEBSD32
 555                                     frame->tf_rip ==
 556                                     (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
 557 #endif
 558                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
 559                                         return;
 560                         } else {
 561                                 if (frame->tf_rip ==
 562                                     (uintptr_t)IDTVEC(fast_syscall) ||
 563 #ifdef COMPAT_FREEBSD32
 564                                     frame->tf_rip ==
 565                                     (uintptr_t)IDTVEC(int0x80_syscall) ||
 566 #endif
 567                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt))
 568                                         return;
 569                         }
 570                         if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
 571                             /* Needed for AMD. */
 572                             frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
 573                                 return;
 574                         /*
 575                          * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
 576                          */
 577                 case T_BPTFLT:
 578                         /*
 579                          * If KDB is enabled, let it handle the debugger trap.
 580                          * Otherwise, debugger traps "can't happen".
 581                          */
 582 #ifdef KDB
 583                         if (kdb_trap(type, dr6, frame))
 584                                 return;
 585 #endif
 586                         break;
 587
 588                 case T_NMI:
 589                         nmi_handle_intr(type, frame);
 590                         return;
 591                 }
 592
 593                 trap_fatal(frame, 0);
 594                 return;
 595         }
 596
 597         /* Translate fault for emulators (e.g. Linux) */
 598         if (*p->p_sysent->sv_transtrap != NULL)
 599                 signo = (*p->p_sysent->sv_transtrap)(signo, type);
 600
 601         ksiginfo_init_trap(&ksi);
 602         ksi.ksi_signo = signo;
 603         ksi.ksi_code = ucode;
 604         ksi.ksi_trapno = type;
 605         ksi.ksi_addr = (void *)addr;
 606         if (uprintf_signal) {
 607                 uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
 608                     "addr 0x%lx rsp 0x%lx rip 0x%lx "
 609                     "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
 610                     p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
 611                     addr, frame->tf_rsp, frame->tf_rip,
 612                     fubyte((void *)(frame->tf_rip + 0)),
 613                     fubyte((void *)(frame->tf_rip + 1)),
 614                     fubyte((void *)(frame->tf_rip + 2)),
 615                     fubyte((void *)(frame->tf_rip + 3)),
 616                     fubyte((void *)(frame->tf_rip + 4)),
 617                     fubyte((void *)(frame->tf_rip + 5)),
 618                     fubyte((void *)(frame->tf_rip + 6)),
 619                     fubyte((void *)(frame->tf_rip + 7)));
 620         }
 621         KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
 622         trapsignal(td, &ksi);
 623
 624 userret:
 625         userret(td, frame);
 626         KASSERT(PCB_USER_FPU(td->td_pcb),
 627             ("Return from trap with kernel FPU ctx leaked"));
 628 }
 629
 630 /*
 631  * Ensure that we ignore any DTrace-induced faults. This function cannot
 632  * be instrumented, so it cannot generate such faults itself.
 633  */
 634 void
 635 trap_check(struct trapframe *frame)
 636 {
 637
 638 #ifdef KDTRACE_HOOKS
 639         if (dtrace_trap_func != NULL &&
 640             (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
 641                 return;
 642 #endif
 643         trap(frame);
 644 }
 645
 646 static bool
 647 trap_is_smap(struct trapframe *frame)
 648 {
 649
 650         /*
 651          * A page fault on a userspace address is classified as
 652          * SMAP-induced if:
 653          * - SMAP is supported;
 654          * - kernel mode accessed present data page;
 655          * - rflags.AC was cleared.
 656          * Kernel must never access user space with rflags.AC cleared
 657          * if SMAP is enabled.
 658          */
 659         return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
 660             (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
 661             PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
 662 }
 663
 664 static bool
 665 trap_is_pti(struct trapframe *frame)
 666 {
 667
 668         return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
 669             pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
 670             PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
 671             (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
 672             (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
 673 }
 674
 675 /*
 676  * Handle all details of a page fault.
 677  * Returns:
 678  * -1 if this fault was fatal, typically from kernel mode
 679  *    (cannot happen, but we need to return something).
 680  * 0  if this fault was handled by updating either the user or kernel
 681  *    page table, execution can continue.
 682  * 1  if this fault was from usermode and it was not handled, a synchronous
 683  *    signal should be delivered to the thread.  *signo returns the signal
 684  *    number, *ucode gives si_code.
 685  */
 686 static int
 687 trap_pfault(struct trapframe *frame, bool usermode, int *signo, int *ucode)
 688 {
 689         struct thread *td;
 690         struct proc *p;
 691         vm_map_t map;
 692         vm_offset_t eva;
 693         int rv;
 694         vm_prot_t ftype;
 695
 696         MPASS(!usermode || (signo != NULL && ucode != NULL));
 697
 698         td = curthread;
 699         p = td->td_proc;
 700         eva = frame->tf_addr;
 701
 702         if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
 703                 /*
 704                  * Due to both processor errata and lazy TLB invalidation when
 705                  * access restrictions are removed from virtual pages, memory
 706                  * accesses that are allowed by the physical mapping layer may
 707                  * nonetheless cause one spurious page fault per virtual page.
 708                  * When the thread is executing a "no faulting" section that
 709                  * is bracketed by vm_fault_{disable,enable}_pagefaults(),
 710                  * every page fault is treated as a spurious page fault,
 711                  * unless it accesses the same virtual address as the most
 712                  * recent page fault within the same "no faulting" section.
 713                  */
 714                 if (td->td_md.md_spurflt_addr != eva ||
 715                     (td->td_pflags & TDP_RESETSPUR) != 0) {
 716                         /*
 717                          * Do nothing to the TLB.  A stale TLB entry is
 718                          * flushed automatically by a page fault.
 719                          */
 720                         td->td_md.md_spurflt_addr = eva;
 721                         td->td_pflags &= ~TDP_RESETSPUR;
 722                         return (0);
 723                 }
 724         } else {
 725                 /*
 726                  * If we get a page fault while in a critical section, then
 727                  * it is most likely a fatal kernel page fault.  The kernel
 728                  * is already going to panic trying to get a sleep lock to
 729                  * do the VM lookup, so just consider it a fatal trap so the
 730                  * kernel can print out a useful trap message and even get
 731                  * to the debugger.
 732                  *
 733                  * If we get a page fault while holding a non-sleepable
 734                  * lock, then it is most likely a fatal kernel page fault.
 735                  * If WITNESS is enabled, then it's going to whine about
 736                  * bogus LORs with various VM locks, so just skip to the
 737                  * fatal trap handling directly.
 738                  */
 739                 if (td->td_critnest != 0 ||
 740                     WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
 741                     "Kernel page fault") != 0) {
 742                         trap_fatal(frame, eva);
 743                         return (-1);
 744                 }
 745         }
 746         if (eva >= VM_MIN_KERNEL_ADDRESS) {
 747                 /*
 748                  * Don't allow user-mode faults in kernel address space.
 749                  */
 750                 if (usermode) {
 751                         *signo = SIGSEGV;
 752                         *ucode = SEGV_MAPERR;
 753                         return (1);
 754                 }
 755
 756                 map = kernel_map;
 757         } else {
 758                 map = &p->p_vmspace->vm_map;
 759
 760                 /*
 761                  * When accessing a usermode address, kernel must be
 762                  * ready to accept the page fault, and provide a
 763                  * handling routine.  Since accessing the address
 764                  * without the handler is a bug, do not try to handle
 765                  * it normally, and panic immediately.
 766                  *
 767                  * If SMAP is enabled, filter SMAP faults also,
 768                  * because illegal access might occur to the mapped
 769                  * user address, causing infinite loop.
 770                  */
 771                 if (!usermode && (td->td_intr_nesting_level != 0 ||
 772                     trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
 773                         trap_fatal(frame, eva);
 774                         return (-1);
 775                 }
 776         }
 777
 778         /*
 779          * If the trap was caused by errant bits in the PTE then panic.
 780          */
 781         if (frame->tf_err & PGEX_RSV) {
 782                 trap_fatal(frame, eva);
 783                 return (-1);
 784         }
 785
 786         /*
 787          * User-mode protection key violation (PKU).  May happen
 788          * either from usermode or from kernel if copyin accessed
 789          * key-protected mapping.
 790          */
 791         if ((frame->tf_err & PGEX_PK) != 0) {
 792                 if (eva > VM_MAXUSER_ADDRESS) {
 793                         trap_fatal(frame, eva);
 794                         return (-1);
 795                 }
 796                 if (usermode) {
 797                         *signo = SIGSEGV;
 798                         *ucode = SEGV_PKUERR;
 799                         return (1);
 800                 }
 801                 goto after_vmfault;
 802         }
 803
 804         /*
 805          * If nx protection of the usermode portion of kernel page
 806          * tables caused trap, panic.
 807          */
 808         if (usermode && trap_is_pti(frame))
 809                 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
 810                     p->p_comm, frame->tf_err);
 811
 812         /*
 813          * PGEX_I is defined only if the execute disable bit capability is
 814          * supported and enabled.
 815          */
 816         if (frame->tf_err & PGEX_W)
 817                 ftype = VM_PROT_WRITE;
 818         else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
 819                 ftype = VM_PROT_EXECUTE;
 820         else
 821                 ftype = VM_PROT_READ;
 822
 823         /* Fault in the page. */
 824         rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
 825         if (rv == KERN_SUCCESS) {
 826 #ifdef HWPMC_HOOKS
 827                 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
 828                         PMC_SOFT_CALL_TF( , , page_fault, all, frame);
 829                         if (ftype == VM_PROT_READ)
 830                                 PMC_SOFT_CALL_TF( , , page_fault, read,
 831                                     frame);
 832                         else
 833                                 PMC_SOFT_CALL_TF( , , page_fault, write,
 834                                     frame);
 835                 }
 836 #endif
 837                 return (0);
 838         }
 839
 840         if (usermode)
 841                 return (1);
 842 after_vmfault:
 843         if (td->td_intr_nesting_level == 0 &&
 844             curpcb->pcb_onfault != NULL) {
 845                 frame->tf_rip = (long)curpcb->pcb_onfault;
 846                 return (0);
 847         }
 848         trap_fatal(frame, eva);
 849         return (-1);
 850 }
 851
 852 static void
 853 trap_fatal(frame, eva)
 854         struct trapframe *frame;
 855         vm_offset_t eva;
 856 {
 857         int code, ss;
 858         u_int type;
 859         struct soft_segment_descriptor softseg;
 860 #ifdef KDB
 861         bool handled;
 862 #endif
 863
 864         code = frame->tf_err;
 865         type = frame->tf_trapno;
 866         sdtossd(&gdt[NGDT * PCPU_GET(cpuid) + IDXSEL(frame->tf_cs & 0xffff)],
 867             &softseg);
 868
 869         printf("\n\nFatal trap %d: %s while in %s mode\n", type,
 870             type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN,
 871             TRAPF_USERMODE(frame) ? "user" : "kernel");
 872 #ifdef SMP
 873         /* two separate prints in case of a trap on an unmapped page */
 874         printf("cpuid = %d; ", PCPU_GET(cpuid));
 875         printf("apic id = %02x\n", PCPU_GET(apic_id));
 876 #endif
 877         if (type == T_PAGEFLT) {
 878                 printf("fault virtual address   = 0x%lx\n", eva);
 879                 printf("fault code              = %s %s %s%s%s, %s\n",
 880                         code & PGEX_U ? "user" : "supervisor",
 881                         code & PGEX_W ? "write" : "read",
 882                         code & PGEX_I ? "instruction" : "data",
 883                         code & PGEX_PK ? " prot key" : "",
 884                         code & PGEX_SGX ? " SGX" : "",
 885                         code & PGEX_RSV ? "reserved bits in PTE" :
 886                         code & PGEX_P ? "protection violation" : "page not present");
 887         }
 888         printf("instruction pointer     = 0x%lx:0x%lx\n",
 889                frame->tf_cs & 0xffff, frame->tf_rip);
 890         ss = frame->tf_ss & 0xffff;
 891         printf("stack pointer           = 0x%x:0x%lx\n", ss, frame->tf_rsp);
 892         printf("frame pointer           = 0x%x:0x%lx\n", ss, frame->tf_rbp);
 893         printf("code segment            = base 0x%lx, limit 0x%lx, type 0x%x\n",
 894                softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
 895         printf("                        = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
 896                softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
 897                softseg.ssd_gran);
 898         printf("processor eflags        = ");
 899         if (frame->tf_rflags & PSL_T)
 900                 printf("trace trap, ");
 901         if (frame->tf_rflags & PSL_I)
 902                 printf("interrupt enabled, ");
 903         if (frame->tf_rflags & PSL_NT)
 904                 printf("nested task, ");
 905         if (frame->tf_rflags & PSL_RF)
 906                 printf("resume, ");
 907         printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
 908         printf("current process         = %d (%s)\n",
 909             curproc->p_pid, curthread->td_name);
 910
 911 #ifdef KDB
 912         if (debugger_on_trap) {
 913                 kdb_why = KDB_WHY_TRAP;
 914                 handled = kdb_trap(type, 0, frame);
 915                 kdb_why = KDB_WHY_UNSET;
 916                 if (handled)
 917                         return;
 918         }
 919 #endif
 920         printf("trap number             = %d\n", type);
 921         panic("%s", type < nitems(trap_msg) ? trap_msg[type] :
 922             "unknown/reserved trap");
 923 }
 924
 925 #ifdef KDTRACE_HOOKS
 926 /*
 927  * Invoke a userspace DTrace hook.  The hook pointer is cleared when no
 928  * userspace probes are enabled, so we must synchronize with DTrace to ensure
 929  * that a trapping thread is able to call the hook before it is cleared.
 930  */
 931 static bool
 932 trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
 933 {
 934         int (*hook)(struct trapframe *);
 935
 936         hook = (int (*)(struct trapframe *))atomic_load_ptr(hookp);
 937         enable_intr();
 938         if (hook != NULL)
 939                 return ((hook)(frame) == 0);
 940         return (false);
 941 }
 942 #endif
 943
 944 /*
 945  * Double fault handler. Called when a fault occurs while writing
 946  * a frame for a trap/exception onto the stack. This usually occurs
 947  * when the stack overflows (such is the case with infinite recursion,
 948  * for example).
 949  */
 950 void
 951 dblfault_handler(struct trapframe *frame)
 952 {
 953 #ifdef KDTRACE_HOOKS
 954         if (dtrace_doubletrap_func != NULL)
 955                 (*dtrace_doubletrap_func)();
 956 #endif
 957         printf("\nFatal double fault\n"
 958             "rip %#lx rsp %#lx rbp %#lx\n"
 959             "rax %#lx rdx %#lx rbx %#lx\n"
 960             "rcx %#lx rsi %#lx rdi %#lx\n"
 961             "r8 %#lx r9 %#lx r10 %#lx\n"
 962             "r11 %#lx r12 %#lx r13 %#lx\n"
 963             "r14 %#lx r15 %#lx rflags %#lx\n"
 964             "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
 965             "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
 966             frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
 967             frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
 968             frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
 969             frame->tf_r8, frame->tf_r9, frame->tf_r10,
 970             frame->tf_r11, frame->tf_r12, frame->tf_r13,
 971             frame->tf_r14, frame->tf_r15, frame->tf_rflags,
 972             frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
 973             frame->tf_fs, frame->tf_gs,
 974             rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
 975 #ifdef SMP
 976         /* two separate prints in case of a trap on an unmapped page */
 977         printf("cpuid = %d; ", PCPU_GET(cpuid));
 978         printf("apic id = %02x\n", PCPU_GET(apic_id));
 979 #endif
 980         panic("double fault");
 981 }
 982
 983 static int __noinline
 984 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
 985 {
 986         struct proc *p;
 987         struct trapframe *frame;
 988         register_t *argp;
 989         caddr_t params;
 990         int reg, regcnt, error;
 991
 992         p = td->td_proc;
 993         frame = td->td_frame;
 994         reg = 0;
 995         regcnt = NARGREGS;
 996
 997         sa->code = frame->tf_rax;
 998
 999         if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
1000                 sa->code = frame->tf_rdi;
1001                 reg++;
1002                 regcnt--;
1003         }
1004         if (p->p_sysent->sv_mask)
1005                 sa->code &= p->p_sysent->sv_mask;
1006
1007         if (sa->code >= p->p_sysent->sv_size)
1008                 sa->callp = &p->p_sysent->sv_table[0];
1009         else
1010                 sa->callp = &p->p_sysent->sv_table[sa->code];
1011
1012         sa->narg = sa->callp->sy_narg;
1013         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
1014         argp = &frame->tf_rdi;
1015         argp += reg;
1016         memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
1017         if (sa->narg > regcnt) {
1018                 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
1019                 error = copyin(params, &sa->args[regcnt],
1020                     (sa->narg - regcnt) * sizeof(sa->args[0]));
1021                 if (__predict_false(error != 0))
1022                         return (error);
1023         }
1024
1025         td->td_retval[0] = 0;
1026         td->td_retval[1] = frame->tf_rdx;
1027
1028         return (0);
1029 }
1030
1031 int
1032 cpu_fetch_syscall_args(struct thread *td)
1033 {
1034         struct proc *p;
1035         struct trapframe *frame;
1036         struct syscall_args *sa;
1037
1038         p = td->td_proc;
1039         frame = td->td_frame;
1040         sa = &td->td_sa;
1041
1042         sa->code = frame->tf_rax;
1043
1044         if (__predict_false(sa->code == SYS_syscall ||
1045             sa->code == SYS___syscall ||
1046             sa->code >= p->p_sysent->sv_size))
1047                 return (cpu_fetch_syscall_args_fallback(td, sa));
1048
1049         sa->callp = &p->p_sysent->sv_table[sa->code];
1050         sa->narg = sa->callp->sy_narg;
1051         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
1052
1053         if (p->p_sysent->sv_mask)
1054                 sa->code &= p->p_sysent->sv_mask;
1055
1056         if (__predict_false(sa->narg > NARGREGS))
1057                 return (cpu_fetch_syscall_args_fallback(td, sa));
1058
1059         memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
1060
1061         td->td_retval[0] = 0;
1062         td->td_retval[1] = frame->tf_rdx;
1063
1064         return (0);
1065 }
1066
1067 #include "../../kern/subr_syscall.c"
1068
1069 static void (*syscall_ret_l1d_flush)(void);
1070 int syscall_ret_l1d_flush_mode;
1071
1072 static void
1073 flush_l1d_hw(void)
1074 {
1075
1076         wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
1077 }
1078
1079 static void __inline
1080 amd64_syscall_ret_flush_l1d_inline(int error)
1081 {
1082         void (*p)(void);
1083
1084         if (error != 0 && error != EEXIST && error != EAGAIN &&
1085             error != EXDEV && error != ENOENT && error != ENOTCONN &&
1086             error != EINPROGRESS) {
1087                 p = syscall_ret_l1d_flush;
1088                 if (p != NULL)
1089                         p();
1090         }
1091 }
1092
1093 void
1094 amd64_syscall_ret_flush_l1d(int error)
1095 {
1096
1097         amd64_syscall_ret_flush_l1d_inline(error);
1098 }
1099
1100 void
1101 amd64_syscall_ret_flush_l1d_recalc(void)
1102 {
1103         bool l1d_hw;
1104
1105         l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
1106 again:
1107         switch (syscall_ret_l1d_flush_mode) {
1108         case 0:
1109                 syscall_ret_l1d_flush = NULL;
1110                 break;
1111         case 1:
1112                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
1113                     flush_l1d_sw_abi;
1114                 break;
1115         case 2:
1116                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
1117                 break;
1118         case 3:
1119                 syscall_ret_l1d_flush = flush_l1d_sw_abi;
1120                 break;
1121         default:
1122                 syscall_ret_l1d_flush_mode = 1;
1123                 goto again;
1124         }
1125 }
1126
1127 static int
1128 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
1129 {
1130         int error, val;
1131
1132         val = syscall_ret_l1d_flush_mode;
1133         error = sysctl_handle_int(oidp, &val, 0, req);
1134         if (error != 0 || req->newptr == NULL)
1135                 return (error);
1136         syscall_ret_l1d_flush_mode = val;
1137         amd64_syscall_ret_flush_l1d_recalc();
1138         return (0);
1139 }
1140 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
1141     CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
1142     machdep_syscall_ret_flush_l1d, "I",
1143     "Flush L1D on syscall return with error (0 - off, 1 - on, "
1144     "2 - use hw only, 3 - use sw only");
1145
1146
1147 /*
1148  * System call handler for native binaries.  The trap frame is already
1149  * set up by the assembler trampoline and a pointer to it is saved in
1150  * td_frame.
1151  */
1152 void
1153 amd64_syscall(struct thread *td, int traced)
1154 {
1155         ksiginfo_t ksi;
1156
1157 #ifdef DIAGNOSTIC
1158         if (!TRAPF_USERMODE(td->td_frame)) {
1159                 panic("syscall");
1160                 /* NOT REACHED */
1161         }
1162 #endif
1163         syscallenter(td);
1164
1165         /*
1166          * Traced syscall.
1167          */
1168         if (__predict_false(traced)) {
1169                 td->td_frame->tf_rflags &= ~PSL_T;
1170                 ksiginfo_init_trap(&ksi);
1171                 ksi.ksi_signo = SIGTRAP;
1172                 ksi.ksi_code = TRAP_TRACE;
1173                 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
1174                 trapsignal(td, &ksi);
1175         }
1176
1177         KASSERT(PCB_USER_FPU(td->td_pcb),
1178             ("System call %s returning with kernel FPU ctx leaked",
1179              syscallname(td->td_proc, td->td_sa.code)));
1180         KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
1181             ("System call %s returning with mangled pcb_save",
1182              syscallname(td->td_proc, td->td_sa.code)));
1183         KASSERT(pmap_not_in_di(),
1184             ("System call %s returning with leaked invl_gen %lu",
1185             syscallname(td->td_proc, td->td_sa.code),
1186             td->td_md.md_invl_gen.gen));
1187
1188         syscallret(td);
1189
1190         /*
1191          * If the user-supplied value of %rip is not a canonical
1192          * address, then some CPUs will trigger a ring 0 #GP during
1193          * the sysret instruction.  However, the fault handler would
1194          * execute in ring 0 with the user's %gs and %rsp which would
1195          * not be safe.  Instead, use the full return path which
1196          * catches the problem safely.
1197          */
1198         if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
1199                 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
1200
1201         amd64_syscall_ret_flush_l1d_inline(td->td_errno);
1202 }