sys/arm64/arm64/machdep.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  */
  27
  28 #include "opt_acpi.h"
  29 #include "opt_platform.h"
  30 #include "opt_ddb.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/buf.h>
  38 #include <sys/bus.h>
  39 #include <sys/cons.h>
  40 #include <sys/cpu.h>
  41 #include <sys/devmap.h>
  42 #include <sys/efi.h>
  43 #include <sys/exec.h>
  44 #include <sys/imgact.h>
  45 #include <sys/kdb.h>
  46 #include <sys/kernel.h>
  47 #include <sys/limits.h>
  48 #include <sys/linker.h>
  49 #include <sys/msgbuf.h>
  50 #include <sys/pcpu.h>
  51 #include <sys/proc.h>
  52 #include <sys/ptrace.h>
  53 #include <sys/reboot.h>
  54 #include <sys/rwlock.h>
  55 #include <sys/sched.h>
  56 #include <sys/signalvar.h>
  57 #include <sys/syscallsubr.h>
  58 #include <sys/sysent.h>
  59 #include <sys/sysproto.h>
  60 #include <sys/ucontext.h>
  61 #include <sys/vdso.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/vm_kern.h>
  65 #include <vm/vm_object.h>
  66 #include <vm/vm_page.h>
  67 #include <vm/pmap.h>
  68 #include <vm/vm_map.h>
  69 #include <vm/vm_pager.h>
  70
  71 #include <machine/armreg.h>
  72 #include <machine/cpu.h>
  73 #include <machine/debug_monitor.h>
  74 #include <machine/kdb.h>
  75 #include <machine/machdep.h>
  76 #include <machine/metadata.h>
  77 #include <machine/md_var.h>
  78 #include <machine/pcb.h>
  79 #include <machine/reg.h>
  80 #include <machine/undefined.h>
  81 #include <machine/vmparam.h>
  82
  83 #include <arm/include/physmem.h>
  84
  85 #ifdef VFP
  86 #include <machine/vfp.h>
  87 #endif
  88
  89 #ifdef DEV_ACPI
  90 #include <contrib/dev/acpica/include/acpi.h>
  91 #include <machine/acpica_machdep.h>
  92 #endif
  93
  94 #ifdef FDT
  95 #include <dev/fdt/fdt_common.h>
  96 #include <dev/ofw/openfirm.h>
  97 #endif
  98
  99
 100 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE;
 101
 102 struct pcpu __pcpu[MAXCPU];
 103
 104 static struct trapframe proc0_tf;
 105
 106 int early_boot = 1;
 107 int cold = 1;
 108
 109 #define PHYSMAP_SIZE    (2 * (VM_PHYSSEG_MAX - 1))
 110 vm_paddr_t physmap[PHYSMAP_SIZE];
 111 u_int physmap_idx;
 112
 113 struct kva_md_info kmi;
 114
 115 int64_t dcache_line_size;       /* The minimum D cache line size */
 116 int64_t icache_line_size;       /* The minimum I cache line size */
 117 int64_t idcache_line_size;      /* The minimum cache line size */
 118 int64_t dczva_line_size;        /* The size of cache line the dc zva zeroes */
 119 int has_pan;
 120
 121 /*
 122  * Physical address of the EFI System Table. Stashed from the metadata hints
 123  * passed into the kernel and used by the EFI code to call runtime services.
 124  */
 125 vm_paddr_t efi_systbl_phys;
 126
 127 /* pagezero_* implementations are provided in support.S */
 128 void pagezero_simple(void *);
 129 void pagezero_cache(void *);
 130
 131 /* pagezero_simple is default pagezero */
 132 void (*pagezero)(void *p) = pagezero_simple;
 133
 134 static void
 135 pan_setup(void)
 136 {
 137         uint64_t id_aa64mfr1;
 138
 139         id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
 140         if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE)
 141                 has_pan = 1;
 142 }
 143
 144 void
 145 pan_enable(void)
 146 {
 147
 148         /*
 149          * The LLVM integrated assembler doesn't understand the PAN
 150          * PSTATE field. Because of this we need to manually create
 151          * the instruction in an asm block. This is equivalent to:
 152          * msr pan, #1
 153          *
 154          * This sets the PAN bit, stopping the kernel from accessing
 155          * memory when userspace can also access it unless the kernel
 156          * uses the userspace load/store instructions.
 157          */
 158         if (has_pan) {
 159                 WRITE_SPECIALREG(sctlr_el1,
 160                     READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN);
 161                 __asm __volatile(".inst 0xd500409f | (0x1 << 8)");
 162         }
 163 }
 164
 165 static void
 166 cpu_startup(void *dummy)
 167 {
 168
 169         undef_init();
 170         identify_cpu();
 171         install_cpu_errata();
 172
 173         vm_ksubmap_init(&kmi);
 174         bufinit();
 175         vm_pager_bufferinit();
 176 }
 177
 178 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 179
 180 int
 181 cpu_idle_wakeup(int cpu)
 182 {
 183
 184         return (0);
 185 }
 186
 187 int
 188 fill_regs(struct thread *td, struct reg *regs)
 189 {
 190         struct trapframe *frame;
 191
 192         frame = td->td_frame;
 193         regs->sp = frame->tf_sp;
 194         regs->lr = frame->tf_lr;
 195         regs->elr = frame->tf_elr;
 196         regs->spsr = frame->tf_spsr;
 197
 198         memcpy(regs->x, frame->tf_x, sizeof(regs->x));
 199
 200         return (0);
 201 }
 202
 203 int
 204 set_regs(struct thread *td, struct reg *regs)
 205 {
 206         struct trapframe *frame;
 207
 208         frame = td->td_frame;
 209         frame->tf_sp = regs->sp;
 210         frame->tf_lr = regs->lr;
 211         frame->tf_elr = regs->elr;
 212         frame->tf_spsr &= ~PSR_FLAGS;
 213         frame->tf_spsr |= regs->spsr & PSR_FLAGS;
 214
 215         memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x));
 216
 217         return (0);
 218 }
 219
 220 int
 221 fill_fpregs(struct thread *td, struct fpreg *regs)
 222 {
 223 #ifdef VFP
 224         struct pcb *pcb;
 225
 226         pcb = td->td_pcb;
 227         if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 228                 /*
 229                  * If we have just been running VFP instructions we will
 230                  * need to save the state to memcpy it below.
 231                  */
 232                 if (td == curthread)
 233                         vfp_save_state(td, pcb);
 234
 235                 KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 236                     ("Called fill_fpregs while the kernel is using the VFP"));
 237                 memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs,
 238                     sizeof(regs->fp_q));
 239                 regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr;
 240                 regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr;
 241         } else
 242 #endif
 243                 memset(regs->fp_q, 0, sizeof(regs->fp_q));
 244         return (0);
 245 }
 246
 247 int
 248 set_fpregs(struct thread *td, struct fpreg *regs)
 249 {
 250 #ifdef VFP
 251         struct pcb *pcb;
 252
 253         pcb = td->td_pcb;
 254         KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 255             ("Called set_fpregs while the kernel is using the VFP"));
 256         memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q));
 257         pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr;
 258         pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr;
 259 #endif
 260         return (0);
 261 }
 262
 263 int
 264 fill_dbregs(struct thread *td, struct dbreg *regs)
 265 {
 266
 267         printf("ARM64TODO: fill_dbregs");
 268         return (EDOOFUS);
 269 }
 270
 271 int
 272 set_dbregs(struct thread *td, struct dbreg *regs)
 273 {
 274
 275         printf("ARM64TODO: set_dbregs");
 276         return (EDOOFUS);
 277 }
 278
 279 #ifdef COMPAT_FREEBSD32
 280 int
 281 fill_regs32(struct thread *td, struct reg32 *regs)
 282 {
 283
 284         printf("ARM64TODO: fill_regs32");
 285         return (EDOOFUS);
 286 }
 287
 288 int
 289 set_regs32(struct thread *td, struct reg32 *regs)
 290 {
 291
 292         printf("ARM64TODO: set_regs32");
 293         return (EDOOFUS);
 294 }
 295
 296 int
 297 fill_fpregs32(struct thread *td, struct fpreg32 *regs)
 298 {
 299
 300         printf("ARM64TODO: fill_fpregs32");
 301         return (EDOOFUS);
 302 }
 303
 304 int
 305 set_fpregs32(struct thread *td, struct fpreg32 *regs)
 306 {
 307
 308         printf("ARM64TODO: set_fpregs32");
 309         return (EDOOFUS);
 310 }
 311
 312 int
 313 fill_dbregs32(struct thread *td, struct dbreg32 *regs)
 314 {
 315
 316         printf("ARM64TODO: fill_dbregs32");
 317         return (EDOOFUS);
 318 }
 319
 320 int
 321 set_dbregs32(struct thread *td, struct dbreg32 *regs)
 322 {
 323
 324         printf("ARM64TODO: set_dbregs32");
 325         return (EDOOFUS);
 326 }
 327 #endif
 328
 329 int
 330 ptrace_set_pc(struct thread *td, u_long addr)
 331 {
 332
 333         printf("ARM64TODO: ptrace_set_pc");
 334         return (EDOOFUS);
 335 }
 336
 337 int
 338 ptrace_single_step(struct thread *td)
 339 {
 340
 341         td->td_frame->tf_spsr |= PSR_SS;
 342         td->td_pcb->pcb_flags |= PCB_SINGLE_STEP;
 343         return (0);
 344 }
 345
 346 int
 347 ptrace_clear_single_step(struct thread *td)
 348 {
 349
 350         td->td_frame->tf_spsr &= ~PSR_SS;
 351         td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
 352         return (0);
 353 }
 354
 355 void
 356 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
 357 {
 358         struct trapframe *tf = td->td_frame;
 359
 360         memset(tf, 0, sizeof(struct trapframe));
 361
 362         tf->tf_x[0] = stack;
 363         tf->tf_sp = STACKALIGN(stack);
 364         tf->tf_lr = imgp->entry_addr;
 365         tf->tf_elr = imgp->entry_addr;
 366 }
 367
 368 /* Sanity check these are the same size, they will be memcpy'd to and fro */
 369 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 370     sizeof((struct gpregs *)0)->gp_x);
 371 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 372     sizeof((struct reg *)0)->x);
 373
 374 int
 375 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret)
 376 {
 377         struct trapframe *tf = td->td_frame;
 378
 379         if (clear_ret & GET_MC_CLEAR_RET) {
 380                 mcp->mc_gpregs.gp_x[0] = 0;
 381                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C;
 382         } else {
 383                 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0];
 384                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr;
 385         }
 386
 387         memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1],
 388             sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1));
 389
 390         mcp->mc_gpregs.gp_sp = tf->tf_sp;
 391         mcp->mc_gpregs.gp_lr = tf->tf_lr;
 392         mcp->mc_gpregs.gp_elr = tf->tf_elr;
 393
 394         return (0);
 395 }
 396
 397 int
 398 set_mcontext(struct thread *td, mcontext_t *mcp)
 399 {
 400         struct trapframe *tf = td->td_frame;
 401         uint32_t spsr;
 402
 403         spsr = mcp->mc_gpregs.gp_spsr;
 404         if ((spsr & PSR_M_MASK) != PSR_M_EL0t ||
 405             (spsr & (PSR_AARCH32 | PSR_F | PSR_I | PSR_A | PSR_D)) != 0)
 406                 return (EINVAL);
 407
 408         memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x));
 409
 410         tf->tf_sp = mcp->mc_gpregs.gp_sp;
 411         tf->tf_lr = mcp->mc_gpregs.gp_lr;
 412         tf->tf_elr = mcp->mc_gpregs.gp_elr;
 413         tf->tf_spsr = mcp->mc_gpregs.gp_spsr;
 414
 415         return (0);
 416 }
 417
 418 static void
 419 get_fpcontext(struct thread *td, mcontext_t *mcp)
 420 {
 421 #ifdef VFP
 422         struct pcb *curpcb;
 423
 424         critical_enter();
 425
 426         curpcb = curthread->td_pcb;
 427
 428         if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 429                 /*
 430                  * If we have just been running VFP instructions we will
 431                  * need to save the state to memcpy it below.
 432                  */
 433                 vfp_save_state(td, curpcb);
 434
 435                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 436                     ("Called get_fpcontext while the kernel is using the VFP"));
 437                 KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
 438                     ("Non-userspace FPU flags set in get_fpcontext"));
 439                 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs,
 440                     sizeof(mcp->mc_fpregs));
 441                 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr;
 442                 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr;
 443                 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
 444                 mcp->mc_flags |= _MC_FP_VALID;
 445         }
 446
 447         critical_exit();
 448 #endif
 449 }
 450
 451 static void
 452 set_fpcontext(struct thread *td, mcontext_t *mcp)
 453 {
 454 #ifdef VFP
 455         struct pcb *curpcb;
 456
 457         critical_enter();
 458
 459         if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
 460                 curpcb = curthread->td_pcb;
 461
 462                 /*
 463                  * Discard any vfp state for the current thread, we
 464                  * are about to override it.
 465                  */
 466                 vfp_discard(td);
 467
 468                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 469                     ("Called set_fpcontext while the kernel is using the VFP"));
 470                 memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q,
 471                     sizeof(mcp->mc_fpregs));
 472                 curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr;
 473                 curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr;
 474                 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK;
 475         }
 476
 477         critical_exit();
 478 #endif
 479 }
 480
 481 void
 482 cpu_idle(int busy)
 483 {
 484
 485         spinlock_enter();
 486         if (!busy)
 487                 cpu_idleclock();
 488         if (!sched_runnable())
 489                 __asm __volatile(
 490                     "dsb sy \n"
 491                     "wfi    \n");
 492         if (!busy)
 493                 cpu_activeclock();
 494         spinlock_exit();
 495 }
 496
 497 void
 498 cpu_halt(void)
 499 {
 500
 501         /* We should have shutdown by now, if not enter a low power sleep */
 502         intr_disable();
 503         while (1) {
 504                 __asm __volatile("wfi");
 505         }
 506 }
 507
 508 /*
 509  * Flush the D-cache for non-DMA I/O so that the I-cache can
 510  * be made coherent later.
 511  */
 512 void
 513 cpu_flush_dcache(void *ptr, size_t len)
 514 {
 515
 516         /* ARM64TODO TBD */
 517 }
 518
 519 /* Get current clock frequency for the given CPU ID. */
 520 int
 521 cpu_est_clockrate(int cpu_id, uint64_t *rate)
 522 {
 523         struct pcpu *pc;
 524
 525         pc = pcpu_find(cpu_id);
 526         if (pc == NULL || rate == NULL)
 527                 return (EINVAL);
 528
 529         if (pc->pc_clock == 0)
 530                 return (EOPNOTSUPP);
 531
 532         *rate = pc->pc_clock;
 533         return (0);
 534 }
 535
 536 void
 537 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 538 {
 539
 540         pcpu->pc_acpi_id = 0xffffffff;
 541 }
 542
 543 void
 544 spinlock_enter(void)
 545 {
 546         struct thread *td;
 547         register_t daif;
 548
 549         td = curthread;
 550         if (td->td_md.md_spinlock_count == 0) {
 551                 daif = intr_disable();
 552                 td->td_md.md_spinlock_count = 1;
 553                 td->td_md.md_saved_daif = daif;
 554         } else
 555                 td->td_md.md_spinlock_count++;
 556         critical_enter();
 557 }
 558
 559 void
 560 spinlock_exit(void)
 561 {
 562         struct thread *td;
 563         register_t daif;
 564
 565         td = curthread;
 566         critical_exit();
 567         daif = td->td_md.md_saved_daif;
 568         td->td_md.md_spinlock_count--;
 569         if (td->td_md.md_spinlock_count == 0)
 570                 intr_restore(daif);
 571 }
 572
 573 #ifndef _SYS_SYSPROTO_H_
 574 struct sigreturn_args {
 575         ucontext_t *ucp;
 576 };
 577 #endif
 578
 579 int
 580 sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 581 {
 582         ucontext_t uc;
 583         int error;
 584
 585         if (uap == NULL)
 586                 return (EFAULT);
 587         if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
 588                 return (EFAULT);
 589
 590         error = set_mcontext(td, &uc.uc_mcontext);
 591         if (error != 0)
 592                 return (error);
 593         set_fpcontext(td, &uc.uc_mcontext);
 594
 595         /* Restore signal mask. */
 596         kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
 597
 598         return (EJUSTRETURN);
 599 }
 600
 601 /*
 602  * Construct a PCB from a trapframe. This is called from kdb_trap() where
 603  * we want to start a backtrace from the function that caused us to enter
 604  * the debugger. We have the context in the trapframe, but base the trace
 605  * on the PCB. The PCB doesn't have to be perfect, as long as it contains
 606  * enough for a backtrace.
 607  */
 608 void
 609 makectx(struct trapframe *tf, struct pcb *pcb)
 610 {
 611         int i;
 612
 613         for (i = 0; i < PCB_LR; i++)
 614                 pcb->pcb_x[i] = tf->tf_x[i];
 615
 616         pcb->pcb_x[PCB_LR] = tf->tf_lr;
 617         pcb->pcb_pc = tf->tf_elr;
 618         pcb->pcb_sp = tf->tf_sp;
 619 }
 620
 621 void
 622 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 623 {
 624         struct thread *td;
 625         struct proc *p;
 626         struct trapframe *tf;
 627         struct sigframe *fp, frame;
 628         struct sigacts *psp;
 629         struct sysentvec *sysent;
 630         int onstack, sig;
 631
 632         td = curthread;
 633         p = td->td_proc;
 634         PROC_LOCK_ASSERT(p, MA_OWNED);
 635
 636         sig = ksi->ksi_signo;
 637         psp = p->p_sigacts;
 638         mtx_assert(&psp->ps_mtx, MA_OWNED);
 639
 640         tf = td->td_frame;
 641         onstack = sigonstack(tf->tf_sp);
 642
 643         CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
 644             catcher, sig);
 645
 646         /* Allocate and validate space for the signal handler context. */
 647         if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
 648             SIGISMEMBER(psp->ps_sigonstack, sig)) {
 649                 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp +
 650                     td->td_sigstk.ss_size);
 651 #if defined(COMPAT_43)
 652                 td->td_sigstk.ss_flags |= SS_ONSTACK;
 653 #endif
 654         } else {
 655                 fp = (struct sigframe *)td->td_frame->tf_sp;
 656         }
 657
 658         /* Make room, keeping the stack aligned */
 659         fp--;
 660         fp = (struct sigframe *)STACKALIGN(fp);
 661
 662         /* Fill in the frame to copy out */
 663         get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
 664         get_fpcontext(td, &frame.sf_uc.uc_mcontext);
 665         frame.sf_si = ksi->ksi_info;
 666         frame.sf_uc.uc_sigmask = *mask;
 667         frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ?
 668             ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE;
 669         frame.sf_uc.uc_stack = td->td_sigstk;
 670         mtx_unlock(&psp->ps_mtx);
 671         PROC_UNLOCK(td->td_proc);
 672
 673         /* Copy the sigframe out to the user's stack. */
 674         if (copyout(&frame, fp, sizeof(*fp)) != 0) {
 675                 /* Process has trashed its stack. Kill it. */
 676                 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
 677                 PROC_LOCK(p);
 678                 sigexit(td, SIGILL);
 679         }
 680
 681         tf->tf_x[0]= sig;
 682         tf->tf_x[1] = (register_t)&fp->sf_si;
 683         tf->tf_x[2] = (register_t)&fp->sf_uc;
 684
 685         tf->tf_elr = (register_t)catcher;
 686         tf->tf_sp = (register_t)fp;
 687         sysent = p->p_sysent;
 688         if (sysent->sv_sigcode_base != 0)
 689                 tf->tf_lr = (register_t)sysent->sv_sigcode_base;
 690         else
 691                 tf->tf_lr = (register_t)(sysent->sv_psstrings -
 692                     *(sysent->sv_szsigcode));
 693
 694         CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr,
 695             tf->tf_sp);
 696
 697         PROC_LOCK(p);
 698         mtx_lock(&psp->ps_mtx);
 699 }
 700
 701 static void
 702 init_proc0(vm_offset_t kstack)
 703 {
 704         struct pcpu *pcpup = &__pcpu[0];
 705
 706         proc_linkup0(&proc0, &thread0);
 707         thread0.td_kstack = kstack;
 708         thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1;
 709         thread0.td_pcb->pcb_fpflags = 0;
 710         thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate;
 711         thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
 712         thread0.td_frame = &proc0_tf;
 713         pcpup->pc_curpcb = thread0.td_pcb;
 714
 715         /* Set the base address of translation table 0. */
 716         thread0.td_proc->p_md.md_l0addr = READ_SPECIALREG(ttbr0_el1);
 717 }
 718
 719 typedef struct {
 720         uint32_t type;
 721         uint64_t phys_start;
 722         uint64_t virt_start;
 723         uint64_t num_pages;
 724         uint64_t attr;
 725 } EFI_MEMORY_DESCRIPTOR;
 726
 727 static int
 728 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
 729     u_int *physmap_idxp)
 730 {
 731         u_int i, insert_idx, _physmap_idx;
 732
 733         _physmap_idx = *physmap_idxp;
 734
 735         if (length == 0)
 736                 return (1);
 737
 738         /*
 739          * Find insertion point while checking for overlap.  Start off by
 740          * assuming the new entry will be added to the end.
 741          */
 742         insert_idx = _physmap_idx;
 743         for (i = 0; i <= _physmap_idx; i += 2) {
 744                 if (base < physmap[i + 1]) {
 745                         if (base + length <= physmap[i]) {
 746                                 insert_idx = i;
 747                                 break;
 748                         }
 749                         if (boothowto & RB_VERBOSE)
 750                                 printf(
 751                     "Overlapping memory regions, ignoring second region\n");
 752                         return (1);
 753                 }
 754         }
 755
 756         /* See if we can prepend to the next entry. */
 757         if (insert_idx <= _physmap_idx &&
 758             base + length == physmap[insert_idx]) {
 759                 physmap[insert_idx] = base;
 760                 return (1);
 761         }
 762
 763         /* See if we can append to the previous entry. */
 764         if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
 765                 physmap[insert_idx - 1] += length;
 766                 return (1);
 767         }
 768
 769         _physmap_idx += 2;
 770         *physmap_idxp = _physmap_idx;
 771         if (_physmap_idx == PHYSMAP_SIZE) {
 772                 printf(
 773                 "Too many segments in the physical address map, giving up\n");
 774                 return (0);
 775         }
 776
 777         /*
 778          * Move the last 'N' entries down to make room for the new
 779          * entry if needed.
 780          */
 781         for (i = _physmap_idx; i > insert_idx; i -= 2) {
 782                 physmap[i] = physmap[i - 2];
 783                 physmap[i + 1] = physmap[i - 1];
 784         }
 785
 786         /* Insert the new entry. */
 787         physmap[insert_idx] = base;
 788         physmap[insert_idx + 1] = base + length;
 789         return (1);
 790 }
 791
 792 #ifdef FDT
 793 static void
 794 add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap,
 795     u_int *physmap_idxp)
 796 {
 797
 798         for (int i = 0; i < mrcnt; i++) {
 799                 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap,
 800                     physmap_idxp))
 801                         break;
 802         }
 803 }
 804 #endif
 805
 806 static void
 807 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
 808     u_int *physmap_idxp)
 809 {
 810         struct efi_md *map, *p;
 811         const char *type;
 812         size_t efisz;
 813         int ndesc, i;
 814
 815         static const char *types[] = {
 816                 "Reserved",
 817                 "LoaderCode",
 818                 "LoaderData",
 819                 "BootServicesCode",
 820                 "BootServicesData",
 821                 "RuntimeServicesCode",
 822                 "RuntimeServicesData",
 823                 "ConventionalMemory",
 824                 "UnusableMemory",
 825                 "ACPIReclaimMemory",
 826                 "ACPIMemoryNVS",
 827                 "MemoryMappedIO",
 828                 "MemoryMappedIOPortSpace",
 829                 "PalCode",
 830                 "PersistentMemory"
 831         };
 832
 833         /*
 834          * Memory map data provided by UEFI via the GetMemoryMap
 835          * Boot Services API.
 836          */
 837         efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
 838         map = (struct efi_md *)((uint8_t *)efihdr + efisz);
 839
 840         if (efihdr->descriptor_size == 0)
 841                 return;
 842         ndesc = efihdr->memory_size / efihdr->descriptor_size;
 843
 844         if (boothowto & RB_VERBOSE)
 845                 printf("%23s %12s %12s %8s %4s\n",
 846                     "Type", "Physical", "Virtual", "#Pages", "Attr");
 847
 848         for (i = 0, p = map; i < ndesc; i++,
 849             p = efi_next_descriptor(p, efihdr->descriptor_size)) {
 850                 if (boothowto & RB_VERBOSE) {
 851                         if (p->md_type < nitems(types))
 852                                 type = types[p->md_type];
 853                         else
 854                                 type = "<INVALID>";
 855                         printf("%23s %012lx %12p %08lx ", type, p->md_phys,
 856                             p->md_virt, p->md_pages);
 857                         if (p->md_attr & EFI_MD_ATTR_UC)
 858                                 printf("UC ");
 859                         if (p->md_attr & EFI_MD_ATTR_WC)
 860                                 printf("WC ");
 861                         if (p->md_attr & EFI_MD_ATTR_WT)
 862                                 printf("WT ");
 863                         if (p->md_attr & EFI_MD_ATTR_WB)
 864                                 printf("WB ");
 865                         if (p->md_attr & EFI_MD_ATTR_UCE)
 866                                 printf("UCE ");
 867                         if (p->md_attr & EFI_MD_ATTR_WP)
 868                                 printf("WP ");
 869                         if (p->md_attr & EFI_MD_ATTR_RP)
 870                                 printf("RP ");
 871                         if (p->md_attr & EFI_MD_ATTR_XP)
 872                                 printf("XP ");
 873                         if (p->md_attr & EFI_MD_ATTR_NV)
 874                                 printf("NV ");
 875                         if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
 876                                 printf("MORE_RELIABLE ");
 877                         if (p->md_attr & EFI_MD_ATTR_RO)
 878                                 printf("RO ");
 879                         if (p->md_attr & EFI_MD_ATTR_RT)
 880                                 printf("RUNTIME");
 881                         printf("\n");
 882                 }
 883
 884                 switch (p->md_type) {
 885                 case EFI_MD_TYPE_CODE:
 886                 case EFI_MD_TYPE_DATA:
 887                 case EFI_MD_TYPE_BS_CODE:
 888                 case EFI_MD_TYPE_BS_DATA:
 889                 case EFI_MD_TYPE_FREE:
 890                         /*
 891                          * We're allowed to use any entry with these types.
 892                          */
 893                         break;
 894                 default:
 895                         continue;
 896                 }
 897
 898                 arm_physmem_hardware_region(p->md_phys,
 899                     p->md_pages * PAGE_SIZE);
 900                 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE),
 901                     physmap, physmap_idxp))
 902                         break;
 903         }
 904 }
 905
 906 #ifdef FDT
 907 static void
 908 try_load_dtb(caddr_t kmdp)
 909 {
 910         vm_offset_t dtbp;
 911
 912         dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
 913         if (dtbp == (vm_offset_t)NULL) {
 914                 printf("ERROR loading DTB\n");
 915                 return;
 916         }
 917
 918         if (OF_install(OFW_FDT, 0) == FALSE)
 919                 panic("Cannot install FDT");
 920
 921         if (OF_init((void *)dtbp) != 0)
 922                 panic("OF_init failed with the found device tree");
 923 }
 924 #endif
 925
 926 static bool
 927 bus_probe(void)
 928 {
 929         bool has_acpi, has_fdt;
 930         char *order, *env;
 931
 932         has_acpi = has_fdt = false;
 933
 934 #ifdef FDT
 935         has_fdt = (OF_peer(0) != 0);
 936 #endif
 937 #ifdef DEV_ACPI
 938         has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0);
 939 #endif
 940
 941         env = kern_getenv("kern.cfg.order");
 942         if (env != NULL) {
 943                 order = env;
 944                 while (order != NULL) {
 945                         if (has_acpi &&
 946                             strncmp(order, "acpi", 4) == 0 &&
 947                             (order[4] == ',' || order[4] == '\0')) {
 948                                 arm64_bus_method = ARM64_BUS_ACPI;
 949                                 break;
 950                         }
 951                         if (has_fdt &&
 952                             strncmp(order, "fdt", 3) == 0 &&
 953                             (order[3] == ',' || order[3] == '\0')) {
 954                                 arm64_bus_method = ARM64_BUS_FDT;
 955                                 break;
 956                         }
 957                         order = strchr(order, ',');
 958                 }
 959                 freeenv(env);
 960
 961                 /* If we set the bus method it is valid */
 962                 if (arm64_bus_method != ARM64_BUS_NONE)
 963                         return (true);
 964         }
 965         /* If no order or an invalid order was set use the default */
 966         if (arm64_bus_method == ARM64_BUS_NONE) {
 967                 if (has_fdt)
 968                         arm64_bus_method = ARM64_BUS_FDT;
 969                 else if (has_acpi)
 970                         arm64_bus_method = ARM64_BUS_ACPI;
 971         }
 972
 973         /*
 974          * If no option was set the default is valid, otherwise we are
 975          * setting one to get cninit() working, then calling panic to tell
 976          * the user about the invalid bus setup.
 977          */
 978         return (env == NULL);
 979 }
 980
 981 static void
 982 cache_setup(void)
 983 {
 984         int dcache_line_shift, icache_line_shift, dczva_line_shift;
 985         uint32_t ctr_el0;
 986         uint32_t dczid_el0;
 987
 988         ctr_el0 = READ_SPECIALREG(ctr_el0);
 989
 990         /* Read the log2 words in each D cache line */
 991         dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
 992         /* Get the D cache line size */
 993         dcache_line_size = sizeof(int) << dcache_line_shift;
 994
 995         /* And the same for the I cache */
 996         icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
 997         icache_line_size = sizeof(int) << icache_line_shift;
 998
 999         idcache_line_size = MIN(dcache_line_size, icache_line_size);
1000
1001         dczid_el0 = READ_SPECIALREG(dczid_el0);
1002
1003         /* Check if dc zva is not prohibited */
1004         if (dczid_el0 & DCZID_DZP)
1005                 dczva_line_size = 0;
1006         else {
1007                 /* Same as with above calculations */
1008                 dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
1009                 dczva_line_size = sizeof(int) << dczva_line_shift;
1010
1011                 /* Change pagezero function */
1012                 pagezero = pagezero_cache;
1013         }
1014 }
1015
1016 void
1017 initarm(struct arm64_bootparams *abp)
1018 {
1019         struct efi_map_header *efihdr;
1020         struct pcpu *pcpup;
1021         char *env;
1022 #ifdef FDT
1023         struct mem_region mem_regions[FDT_MEM_REGIONS];
1024         int mem_regions_sz;
1025 #endif
1026         vm_offset_t lastaddr;
1027         caddr_t kmdp;
1028         bool valid;
1029
1030         /* Set the module data location */
1031         preload_metadata = (caddr_t)(uintptr_t)(abp->modulep);
1032
1033         /* Find the kernel address */
1034         kmdp = preload_search_by_type("elf kernel");
1035         if (kmdp == NULL)
1036                 kmdp = preload_search_by_type("elf64 kernel");
1037
1038         boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
1039         init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0);
1040
1041 #ifdef FDT
1042         try_load_dtb(kmdp);
1043 #endif
1044
1045         efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);
1046
1047         /* Find the address to start allocating from */
1048         lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
1049
1050         /* Load the physical memory ranges */
1051         physmap_idx = 0;
1052         efihdr = (struct efi_map_header *)preload_search_info(kmdp,
1053             MODINFO_METADATA | MODINFOMD_EFI_MAP);
1054         if (efihdr != NULL)
1055                 add_efi_map_entries(efihdr, physmap, &physmap_idx);
1056 #ifdef FDT
1057         else {
1058                 /* Grab physical memory regions information from device tree. */
1059                 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,
1060                     NULL) != 0)
1061                         panic("Cannot get physical memory regions");
1062                 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap,
1063                     &physmap_idx);
1064                 arm_physmem_hardware_regions(mem_regions, mem_regions_sz);
1065         }
1066         if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0)
1067                 arm_physmem_exclude_regions(mem_regions, mem_regions_sz,
1068                     EXFLAG_NODUMP | EXFLAG_NOALLOC);
1069 #endif
1070
1071         /* Set the pcpu data, this is needed by pmap_bootstrap */
1072         pcpup = &__pcpu[0];
1073         pcpu_init(pcpup, 0, sizeof(struct pcpu));
1074
1075         /*
1076          * Set the pcpu pointer with a backup in tpidr_el1 to be
1077          * loaded when entering the kernel from userland.
1078          */
1079         __asm __volatile(
1080             "mov x18, %0 \n"
1081             "msr tpidr_el1, %0" :: "r"(pcpup));
1082
1083         PCPU_SET(curthread, &thread0);
1084
1085         /* Do basic tuning, hz etc */
1086         init_param1();
1087
1088         cache_setup();
1089         pan_setup();
1090
1091         /* Bootstrap enough of pmap  to enter the kernel proper */
1092         pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt,
1093             KERNBASE - abp->kern_delta, lastaddr - KERNBASE);
1094         arm_physmem_init_kernel_globals();
1095
1096         devmap_bootstrap(0, NULL);
1097
1098         valid = bus_probe();
1099
1100         cninit();
1101
1102         if (!valid)
1103                 panic("Invalid bus configuration: %s",
1104                     kern_getenv("kern.cfg.order"));
1105
1106         init_proc0(abp->kern_stack);
1107         msgbufinit(msgbufp, msgbufsize);
1108         mutex_init();
1109         init_param2(physmem);
1110
1111         dbg_init();
1112         kdb_init();
1113         pan_enable();
1114
1115         env = kern_getenv("kernelname");
1116         if (env != NULL)
1117                 strlcpy(kernelname, env, sizeof(kernelname));
1118
1119         if (bootverbose)
1120                 arm_physmem_print_tables();
1121
1122         early_boot = 0;
1123 }
1124
1125 void
1126 dbg_init(void)
1127 {
1128
1129         /* Clear OS lock */
1130         WRITE_SPECIALREG(OSLAR_EL1, 0);
1131
1132         /* This permits DDB to use debug registers for watchpoints. */
1133         dbg_monitor_init();
1134
1135         /* TODO: Eventually will need to initialize debug registers here. */
1136 }
1137
1138 #ifdef DDB
1139 #include <ddb/ddb.h>
1140
1141 DB_SHOW_COMMAND(specialregs, db_show_spregs)
1142 {
1143 #define PRINT_REG(reg)  \
1144     db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg))
1145
1146         PRINT_REG(actlr_el1);
1147         PRINT_REG(afsr0_el1);
1148         PRINT_REG(afsr1_el1);
1149         PRINT_REG(aidr_el1);
1150         PRINT_REG(amair_el1);
1151         PRINT_REG(ccsidr_el1);
1152         PRINT_REG(clidr_el1);
1153         PRINT_REG(contextidr_el1);
1154         PRINT_REG(cpacr_el1);
1155         PRINT_REG(csselr_el1);
1156         PRINT_REG(ctr_el0);
1157         PRINT_REG(currentel);
1158         PRINT_REG(daif);
1159         PRINT_REG(dczid_el0);
1160         PRINT_REG(elr_el1);
1161         PRINT_REG(esr_el1);
1162         PRINT_REG(far_el1);
1163 #if 0
1164         /* ARM64TODO: Enable VFP before reading floating-point registers */
1165         PRINT_REG(fpcr);
1166         PRINT_REG(fpsr);
1167 #endif
1168         PRINT_REG(id_aa64afr0_el1);
1169         PRINT_REG(id_aa64afr1_el1);
1170         PRINT_REG(id_aa64dfr0_el1);
1171         PRINT_REG(id_aa64dfr1_el1);
1172         PRINT_REG(id_aa64isar0_el1);
1173         PRINT_REG(id_aa64isar1_el1);
1174         PRINT_REG(id_aa64pfr0_el1);
1175         PRINT_REG(id_aa64pfr1_el1);
1176         PRINT_REG(id_afr0_el1);
1177         PRINT_REG(id_dfr0_el1);
1178         PRINT_REG(id_isar0_el1);
1179         PRINT_REG(id_isar1_el1);
1180         PRINT_REG(id_isar2_el1);
1181         PRINT_REG(id_isar3_el1);
1182         PRINT_REG(id_isar4_el1);
1183         PRINT_REG(id_isar5_el1);
1184         PRINT_REG(id_mmfr0_el1);
1185         PRINT_REG(id_mmfr1_el1);
1186         PRINT_REG(id_mmfr2_el1);
1187         PRINT_REG(id_mmfr3_el1);
1188 #if 0
1189         /* Missing from llvm */
1190         PRINT_REG(id_mmfr4_el1);
1191 #endif
1192         PRINT_REG(id_pfr0_el1);
1193         PRINT_REG(id_pfr1_el1);
1194         PRINT_REG(isr_el1);
1195         PRINT_REG(mair_el1);
1196         PRINT_REG(midr_el1);
1197         PRINT_REG(mpidr_el1);
1198         PRINT_REG(mvfr0_el1);
1199         PRINT_REG(mvfr1_el1);
1200         PRINT_REG(mvfr2_el1);
1201         PRINT_REG(revidr_el1);
1202         PRINT_REG(sctlr_el1);
1203         PRINT_REG(sp_el0);
1204         PRINT_REG(spsel);
1205         PRINT_REG(spsr_el1);
1206         PRINT_REG(tcr_el1);
1207         PRINT_REG(tpidr_el0);
1208         PRINT_REG(tpidr_el1);
1209         PRINT_REG(tpidrro_el0);
1210         PRINT_REG(ttbr0_el1);
1211         PRINT_REG(ttbr1_el1);
1212         PRINT_REG(vbar_el1);
1213 #undef PRINT_REG
1214 }
1215
1216 DB_SHOW_COMMAND(vtop, db_show_vtop)
1217 {
1218         uint64_t phys;
1219
1220         if (have_addr) {
1221                 phys = arm64_address_translate_s1e1r(addr);
1222                 db_printf("EL1 physical address reg (read):  0x%016lx\n", phys);
1223                 phys = arm64_address_translate_s1e1w(addr);
1224                 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys);
1225                 phys = arm64_address_translate_s1e0r(addr);
1226                 db_printf("EL0 physical address reg (read):  0x%016lx\n", phys);
1227                 phys = arm64_address_translate_s1e0w(addr);
1228                 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys);
1229         } else
1230                 db_printf("show vtop <virt_addr>\n");
1231 }
1232 #endif