sys/arm64/arm64/machdep.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  */
  27
  28 #include "opt_acpi.h"
  29 #include "opt_platform.h"
  30 #include "opt_ddb.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/buf.h>
  38 #include <sys/bus.h>
  39 #include <sys/cons.h>
  40 #include <sys/cpu.h>
  41 #include <sys/devmap.h>
  42 #include <sys/efi.h>
  43 #include <sys/exec.h>
  44 #include <sys/imgact.h>
  45 #include <sys/kdb.h>
  46 #include <sys/kernel.h>
  47 #include <sys/limits.h>
  48 #include <sys/linker.h>
  49 #include <sys/msgbuf.h>
  50 #include <sys/pcpu.h>
  51 #include <sys/proc.h>
  52 #include <sys/ptrace.h>
  53 #include <sys/reboot.h>
  54 #include <sys/rwlock.h>
  55 #include <sys/sched.h>
  56 #include <sys/signalvar.h>
  57 #include <sys/syscallsubr.h>
  58 #include <sys/sysent.h>
  59 #include <sys/sysproto.h>
  60 #include <sys/ucontext.h>
  61 #include <sys/vdso.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/vm_kern.h>
  65 #include <vm/vm_object.h>
  66 #include <vm/vm_page.h>
  67 #include <vm/pmap.h>
  68 #include <vm/vm_map.h>
  69 #include <vm/vm_pager.h>
  70
  71 #include <machine/armreg.h>
  72 #include <machine/cpu.h>
  73 #include <machine/debug_monitor.h>
  74 #include <machine/kdb.h>
  75 #include <machine/machdep.h>
  76 #include <machine/metadata.h>
  77 #include <machine/md_var.h>
  78 #include <machine/pcb.h>
  79 #include <machine/reg.h>
  80 #include <machine/undefined.h>
  81 #include <machine/vmparam.h>
  82
  83 #ifdef VFP
  84 #include <machine/vfp.h>
  85 #endif
  86
  87 #ifdef DEV_ACPI
  88 #include <contrib/dev/acpica/include/acpi.h>
  89 #include <machine/acpica_machdep.h>
  90 #endif
  91
  92 #ifdef FDT
  93 #include <dev/fdt/fdt_common.h>
  94 #include <dev/ofw/openfirm.h>
  95 #endif
  96
  97
  98 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE;
  99
 100 struct pcpu __pcpu[MAXCPU];
 101
 102 static struct trapframe proc0_tf;
 103
 104 vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2];
 105 vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2];
 106
 107 int early_boot = 1;
 108 int cold = 1;
 109 long realmem = 0;
 110 long Maxmem = 0;
 111
 112 #define PHYSMAP_SIZE    (2 * (VM_PHYSSEG_MAX - 1))
 113 vm_paddr_t physmap[PHYSMAP_SIZE];
 114 u_int physmap_idx;
 115
 116 struct kva_md_info kmi;
 117
 118 int64_t dcache_line_size;       /* The minimum D cache line size */
 119 int64_t icache_line_size;       /* The minimum I cache line size */
 120 int64_t idcache_line_size;      /* The minimum cache line size */
 121 int64_t dczva_line_size;        /* The size of cache line the dc zva zeroes */
 122 int has_pan;
 123
 124 /*
 125  * Physical address of the EFI System Table. Stashed from the metadata hints
 126  * passed into the kernel and used by the EFI code to call runtime services.
 127  */
 128 vm_paddr_t efi_systbl_phys;
 129
 130 /* pagezero_* implementations are provided in support.S */
 131 void pagezero_simple(void *);
 132 void pagezero_cache(void *);
 133
 134 /* pagezero_simple is default pagezero */
 135 void (*pagezero)(void *p) = pagezero_simple;
 136
 137 static void
 138 pan_setup(void)
 139 {
 140         uint64_t id_aa64mfr1;
 141
 142         id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
 143         if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE)
 144                 has_pan = 1;
 145 }
 146
 147 void
 148 pan_enable(void)
 149 {
 150
 151         /*
 152          * The LLVM integrated assembler doesn't understand the PAN
 153          * PSTATE field. Because of this we need to manually create
 154          * the instruction in an asm block. This is equivalent to:
 155          * msr pan, #1
 156          *
 157          * This sets the PAN bit, stopping the kernel from accessing
 158          * memory when userspace can also access it unless the kernel
 159          * uses the userspace load/store instructions.
 160          */
 161         if (has_pan) {
 162                 WRITE_SPECIALREG(sctlr_el1,
 163                     READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN);
 164                 __asm __volatile(".inst 0xd500409f | (0x1 << 8)");
 165         }
 166 }
 167
 168 static void
 169 cpu_startup(void *dummy)
 170 {
 171
 172         undef_init();
 173         identify_cpu();
 174
 175         vm_ksubmap_init(&kmi);
 176         bufinit();
 177         vm_pager_bufferinit();
 178 }
 179
 180 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 181
 182 int
 183 cpu_idle_wakeup(int cpu)
 184 {
 185
 186         return (0);
 187 }
 188
 189 int
 190 fill_regs(struct thread *td, struct reg *regs)
 191 {
 192         struct trapframe *frame;
 193
 194         frame = td->td_frame;
 195         regs->sp = frame->tf_sp;
 196         regs->lr = frame->tf_lr;
 197         regs->elr = frame->tf_elr;
 198         regs->spsr = frame->tf_spsr;
 199
 200         memcpy(regs->x, frame->tf_x, sizeof(regs->x));
 201
 202         return (0);
 203 }
 204
 205 int
 206 set_regs(struct thread *td, struct reg *regs)
 207 {
 208         struct trapframe *frame;
 209
 210         frame = td->td_frame;
 211         frame->tf_sp = regs->sp;
 212         frame->tf_lr = regs->lr;
 213         frame->tf_elr = regs->elr;
 214         frame->tf_spsr = regs->spsr;
 215
 216         memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x));
 217
 218         return (0);
 219 }
 220
 221 int
 222 fill_fpregs(struct thread *td, struct fpreg *regs)
 223 {
 224 #ifdef VFP
 225         struct pcb *pcb;
 226
 227         pcb = td->td_pcb;
 228         if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 229                 /*
 230                  * If we have just been running VFP instructions we will
 231                  * need to save the state to memcpy it below.
 232                  */
 233                 if (td == curthread)
 234                         vfp_save_state(td, pcb);
 235
 236                 KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 237                     ("Called fill_fpregs while the kernel is using the VFP"));
 238                 memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs,
 239                     sizeof(regs->fp_q));
 240                 regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr;
 241                 regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr;
 242         } else
 243 #endif
 244                 memset(regs->fp_q, 0, sizeof(regs->fp_q));
 245         return (0);
 246 }
 247
 248 int
 249 set_fpregs(struct thread *td, struct fpreg *regs)
 250 {
 251 #ifdef VFP
 252         struct pcb *pcb;
 253
 254         pcb = td->td_pcb;
 255         KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 256             ("Called set_fpregs while the kernel is using the VFP"));
 257         memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q));
 258         pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr;
 259         pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr;
 260 #endif
 261         return (0);
 262 }
 263
 264 int
 265 fill_dbregs(struct thread *td, struct dbreg *regs)
 266 {
 267
 268         printf("ARM64TODO: fill_dbregs");
 269         return (EDOOFUS);
 270 }
 271
 272 int
 273 set_dbregs(struct thread *td, struct dbreg *regs)
 274 {
 275
 276         printf("ARM64TODO: set_dbregs");
 277         return (EDOOFUS);
 278 }
 279
 280 int
 281 ptrace_set_pc(struct thread *td, u_long addr)
 282 {
 283
 284         printf("ARM64TODO: ptrace_set_pc");
 285         return (EDOOFUS);
 286 }
 287
 288 int
 289 ptrace_single_step(struct thread *td)
 290 {
 291
 292         td->td_frame->tf_spsr |= PSR_SS;
 293         td->td_pcb->pcb_flags |= PCB_SINGLE_STEP;
 294         return (0);
 295 }
 296
 297 int
 298 ptrace_clear_single_step(struct thread *td)
 299 {
 300
 301         td->td_frame->tf_spsr &= ~PSR_SS;
 302         td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
 303         return (0);
 304 }
 305
 306 void
 307 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
 308 {
 309         struct trapframe *tf = td->td_frame;
 310
 311         memset(tf, 0, sizeof(struct trapframe));
 312
 313         /*
 314          * We need to set x0 for init as it doesn't call
 315          * cpu_set_syscall_retval to copy the value. We also
 316          * need to set td_retval for the cases where we do.
 317          */
 318         tf->tf_x[0] = td->td_retval[0] = stack;
 319         tf->tf_sp = STACKALIGN(stack);
 320         tf->tf_lr = imgp->entry_addr;
 321         tf->tf_elr = imgp->entry_addr;
 322 }
 323
 324 /* Sanity check these are the same size, they will be memcpy'd to and fro */
 325 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 326     sizeof((struct gpregs *)0)->gp_x);
 327 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 328     sizeof((struct reg *)0)->x);
 329
 330 int
 331 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret)
 332 {
 333         struct trapframe *tf = td->td_frame;
 334
 335         if (clear_ret & GET_MC_CLEAR_RET) {
 336                 mcp->mc_gpregs.gp_x[0] = 0;
 337                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C;
 338         } else {
 339                 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0];
 340                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr;
 341         }
 342
 343         memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1],
 344             sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1));
 345
 346         mcp->mc_gpregs.gp_sp = tf->tf_sp;
 347         mcp->mc_gpregs.gp_lr = tf->tf_lr;
 348         mcp->mc_gpregs.gp_elr = tf->tf_elr;
 349
 350         return (0);
 351 }
 352
 353 int
 354 set_mcontext(struct thread *td, mcontext_t *mcp)
 355 {
 356         struct trapframe *tf = td->td_frame;
 357
 358         memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x));
 359
 360         tf->tf_sp = mcp->mc_gpregs.gp_sp;
 361         tf->tf_lr = mcp->mc_gpregs.gp_lr;
 362         tf->tf_elr = mcp->mc_gpregs.gp_elr;
 363         tf->tf_spsr = mcp->mc_gpregs.gp_spsr;
 364
 365         return (0);
 366 }
 367
 368 static void
 369 get_fpcontext(struct thread *td, mcontext_t *mcp)
 370 {
 371 #ifdef VFP
 372         struct pcb *curpcb;
 373
 374         critical_enter();
 375
 376         curpcb = curthread->td_pcb;
 377
 378         if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 379                 /*
 380                  * If we have just been running VFP instructions we will
 381                  * need to save the state to memcpy it below.
 382                  */
 383                 vfp_save_state(td, curpcb);
 384
 385                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 386                     ("Called get_fpcontext while the kernel is using the VFP"));
 387                 KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
 388                     ("Non-userspace FPU flags set in get_fpcontext"));
 389                 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs,
 390                     sizeof(mcp->mc_fpregs));
 391                 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr;
 392                 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr;
 393                 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
 394                 mcp->mc_flags |= _MC_FP_VALID;
 395         }
 396
 397         critical_exit();
 398 #endif
 399 }
 400
 401 static void
 402 set_fpcontext(struct thread *td, mcontext_t *mcp)
 403 {
 404 #ifdef VFP
 405         struct pcb *curpcb;
 406
 407         critical_enter();
 408
 409         if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
 410                 curpcb = curthread->td_pcb;
 411
 412                 /*
 413                  * Discard any vfp state for the current thread, we
 414                  * are about to override it.
 415                  */
 416                 vfp_discard(td);
 417
 418                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 419                     ("Called set_fpcontext while the kernel is using the VFP"));
 420                 memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q,
 421                     sizeof(mcp->mc_fpregs));
 422                 curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr;
 423                 curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr;
 424                 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK;
 425         }
 426
 427         critical_exit();
 428 #endif
 429 }
 430
 431 void
 432 cpu_idle(int busy)
 433 {
 434
 435         spinlock_enter();
 436         if (!busy)
 437                 cpu_idleclock();
 438         if (!sched_runnable())
 439                 __asm __volatile(
 440                     "dsb sy \n"
 441                     "wfi    \n");
 442         if (!busy)
 443                 cpu_activeclock();
 444         spinlock_exit();
 445 }
 446
 447 void
 448 cpu_halt(void)
 449 {
 450
 451         /* We should have shutdown by now, if not enter a low power sleep */
 452         intr_disable();
 453         while (1) {
 454                 __asm __volatile("wfi");
 455         }
 456 }
 457
 458 /*
 459  * Flush the D-cache for non-DMA I/O so that the I-cache can
 460  * be made coherent later.
 461  */
 462 void
 463 cpu_flush_dcache(void *ptr, size_t len)
 464 {
 465
 466         /* ARM64TODO TBD */
 467 }
 468
 469 /* Get current clock frequency for the given CPU ID. */
 470 int
 471 cpu_est_clockrate(int cpu_id, uint64_t *rate)
 472 {
 473         struct pcpu *pc;
 474
 475         pc = pcpu_find(cpu_id);
 476         if (pc == NULL || rate == NULL)
 477                 return (EINVAL);
 478
 479         if (pc->pc_clock == 0)
 480                 return (EOPNOTSUPP);
 481
 482         *rate = pc->pc_clock;
 483         return (0);
 484 }
 485
 486 void
 487 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 488 {
 489
 490         pcpu->pc_acpi_id = 0xffffffff;
 491 }
 492
 493 void
 494 spinlock_enter(void)
 495 {
 496         struct thread *td;
 497         register_t daif;
 498
 499         td = curthread;
 500         if (td->td_md.md_spinlock_count == 0) {
 501                 daif = intr_disable();
 502                 td->td_md.md_spinlock_count = 1;
 503                 td->td_md.md_saved_daif = daif;
 504         } else
 505                 td->td_md.md_spinlock_count++;
 506         critical_enter();
 507 }
 508
 509 void
 510 spinlock_exit(void)
 511 {
 512         struct thread *td;
 513         register_t daif;
 514
 515         td = curthread;
 516         critical_exit();
 517         daif = td->td_md.md_saved_daif;
 518         td->td_md.md_spinlock_count--;
 519         if (td->td_md.md_spinlock_count == 0)
 520                 intr_restore(daif);
 521 }
 522
 523 #ifndef _SYS_SYSPROTO_H_
 524 struct sigreturn_args {
 525         ucontext_t *ucp;
 526 };
 527 #endif
 528
 529 int
 530 sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 531 {
 532         ucontext_t uc;
 533         uint32_t spsr;
 534
 535         if (uap == NULL)
 536                 return (EFAULT);
 537         if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
 538                 return (EFAULT);
 539
 540         spsr = uc.uc_mcontext.mc_gpregs.gp_spsr;
 541         if ((spsr & PSR_M_MASK) != PSR_M_EL0t ||
 542             (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0)
 543                 return (EINVAL);
 544
 545         set_mcontext(td, &uc.uc_mcontext);
 546         set_fpcontext(td, &uc.uc_mcontext);
 547
 548         /* Restore signal mask. */
 549         kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
 550
 551         return (EJUSTRETURN);
 552 }
 553
 554 /*
 555  * Construct a PCB from a trapframe. This is called from kdb_trap() where
 556  * we want to start a backtrace from the function that caused us to enter
 557  * the debugger. We have the context in the trapframe, but base the trace
 558  * on the PCB. The PCB doesn't have to be perfect, as long as it contains
 559  * enough for a backtrace.
 560  */
 561 void
 562 makectx(struct trapframe *tf, struct pcb *pcb)
 563 {
 564         int i;
 565
 566         for (i = 0; i < PCB_LR; i++)
 567                 pcb->pcb_x[i] = tf->tf_x[i];
 568
 569         pcb->pcb_x[PCB_LR] = tf->tf_lr;
 570         pcb->pcb_pc = tf->tf_elr;
 571         pcb->pcb_sp = tf->tf_sp;
 572 }
 573
 574 void
 575 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 576 {
 577         struct thread *td;
 578         struct proc *p;
 579         struct trapframe *tf;
 580         struct sigframe *fp, frame;
 581         struct sigacts *psp;
 582         struct sysentvec *sysent;
 583         int code, onstack, sig;
 584
 585         td = curthread;
 586         p = td->td_proc;
 587         PROC_LOCK_ASSERT(p, MA_OWNED);
 588
 589         sig = ksi->ksi_signo;
 590         code = ksi->ksi_code;
 591         psp = p->p_sigacts;
 592         mtx_assert(&psp->ps_mtx, MA_OWNED);
 593
 594         tf = td->td_frame;
 595         onstack = sigonstack(tf->tf_sp);
 596
 597         CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
 598             catcher, sig);
 599
 600         /* Allocate and validate space for the signal handler context. */
 601         if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
 602             SIGISMEMBER(psp->ps_sigonstack, sig)) {
 603                 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp +
 604                     td->td_sigstk.ss_size);
 605 #if defined(COMPAT_43)
 606                 td->td_sigstk.ss_flags |= SS_ONSTACK;
 607 #endif
 608         } else {
 609                 fp = (struct sigframe *)td->td_frame->tf_sp;
 610         }
 611
 612         /* Make room, keeping the stack aligned */
 613         fp--;
 614         fp = (struct sigframe *)STACKALIGN(fp);
 615
 616         /* Fill in the frame to copy out */
 617         get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
 618         get_fpcontext(td, &frame.sf_uc.uc_mcontext);
 619         frame.sf_si = ksi->ksi_info;
 620         frame.sf_uc.uc_sigmask = *mask;
 621         frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ?
 622             ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE;
 623         frame.sf_uc.uc_stack = td->td_sigstk;
 624         mtx_unlock(&psp->ps_mtx);
 625         PROC_UNLOCK(td->td_proc);
 626
 627         /* Copy the sigframe out to the user's stack. */
 628         if (copyout(&frame, fp, sizeof(*fp)) != 0) {
 629                 /* Process has trashed its stack. Kill it. */
 630                 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
 631                 PROC_LOCK(p);
 632                 sigexit(td, SIGILL);
 633         }
 634
 635         tf->tf_x[0]= sig;
 636         tf->tf_x[1] = (register_t)&fp->sf_si;
 637         tf->tf_x[2] = (register_t)&fp->sf_uc;
 638
 639         tf->tf_elr = (register_t)catcher;
 640         tf->tf_sp = (register_t)fp;
 641         sysent = p->p_sysent;
 642         if (sysent->sv_sigcode_base != 0)
 643                 tf->tf_lr = (register_t)sysent->sv_sigcode_base;
 644         else
 645                 tf->tf_lr = (register_t)(sysent->sv_psstrings -
 646                     *(sysent->sv_szsigcode));
 647
 648         CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr,
 649             tf->tf_sp);
 650
 651         PROC_LOCK(p);
 652         mtx_lock(&psp->ps_mtx);
 653 }
 654
 655 static void
 656 init_proc0(vm_offset_t kstack)
 657 {
 658         struct pcpu *pcpup = &__pcpu[0];
 659
 660         proc_linkup0(&proc0, &thread0);
 661         thread0.td_kstack = kstack;
 662         thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1;
 663         thread0.td_pcb->pcb_fpflags = 0;
 664         thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate;
 665         thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
 666         thread0.td_frame = &proc0_tf;
 667         pcpup->pc_curpcb = thread0.td_pcb;
 668 }
 669
 670 typedef struct {
 671         uint32_t type;
 672         uint64_t phys_start;
 673         uint64_t virt_start;
 674         uint64_t num_pages;
 675         uint64_t attr;
 676 } EFI_MEMORY_DESCRIPTOR;
 677
 678 static int
 679 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
 680     u_int *physmap_idxp)
 681 {
 682         u_int i, insert_idx, _physmap_idx;
 683
 684         _physmap_idx = *physmap_idxp;
 685
 686         if (length == 0)
 687                 return (1);
 688
 689         /*
 690          * Find insertion point while checking for overlap.  Start off by
 691          * assuming the new entry will be added to the end.
 692          */
 693         insert_idx = _physmap_idx;
 694         for (i = 0; i <= _physmap_idx; i += 2) {
 695                 if (base < physmap[i + 1]) {
 696                         if (base + length <= physmap[i]) {
 697                                 insert_idx = i;
 698                                 break;
 699                         }
 700                         if (boothowto & RB_VERBOSE)
 701                                 printf(
 702                     "Overlapping memory regions, ignoring second region\n");
 703                         return (1);
 704                 }
 705         }
 706
 707         /* See if we can prepend to the next entry. */
 708         if (insert_idx <= _physmap_idx &&
 709             base + length == physmap[insert_idx]) {
 710                 physmap[insert_idx] = base;
 711                 return (1);
 712         }
 713
 714         /* See if we can append to the previous entry. */
 715         if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
 716                 physmap[insert_idx - 1] += length;
 717                 return (1);
 718         }
 719
 720         _physmap_idx += 2;
 721         *physmap_idxp = _physmap_idx;
 722         if (_physmap_idx == PHYSMAP_SIZE) {
 723                 printf(
 724                 "Too many segments in the physical address map, giving up\n");
 725                 return (0);
 726         }
 727
 728         /*
 729          * Move the last 'N' entries down to make room for the new
 730          * entry if needed.
 731          */
 732         for (i = _physmap_idx; i > insert_idx; i -= 2) {
 733                 physmap[i] = physmap[i - 2];
 734                 physmap[i + 1] = physmap[i - 1];
 735         }
 736
 737         /* Insert the new entry. */
 738         physmap[insert_idx] = base;
 739         physmap[insert_idx + 1] = base + length;
 740         return (1);
 741 }
 742
 743 #ifdef FDT
 744 static void
 745 add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap,
 746     u_int *physmap_idxp)
 747 {
 748
 749         for (int i = 0; i < mrcnt; i++) {
 750                 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap,
 751                     physmap_idxp))
 752                         break;
 753         }
 754 }
 755 #endif
 756
 757 static void
 758 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
 759     u_int *physmap_idxp)
 760 {
 761         struct efi_md *map, *p;
 762         const char *type;
 763         size_t efisz;
 764         int ndesc, i;
 765
 766         static const char *types[] = {
 767                 "Reserved",
 768                 "LoaderCode",
 769                 "LoaderData",
 770                 "BootServicesCode",
 771                 "BootServicesData",
 772                 "RuntimeServicesCode",
 773                 "RuntimeServicesData",
 774                 "ConventionalMemory",
 775                 "UnusableMemory",
 776                 "ACPIReclaimMemory",
 777                 "ACPIMemoryNVS",
 778                 "MemoryMappedIO",
 779                 "MemoryMappedIOPortSpace",
 780                 "PalCode",
 781                 "PersistentMemory"
 782         };
 783
 784         /*
 785          * Memory map data provided by UEFI via the GetMemoryMap
 786          * Boot Services API.
 787          */
 788         efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
 789         map = (struct efi_md *)((uint8_t *)efihdr + efisz);
 790
 791         if (efihdr->descriptor_size == 0)
 792                 return;
 793         ndesc = efihdr->memory_size / efihdr->descriptor_size;
 794
 795         if (boothowto & RB_VERBOSE)
 796                 printf("%23s %12s %12s %8s %4s\n",
 797                     "Type", "Physical", "Virtual", "#Pages", "Attr");
 798
 799         for (i = 0, p = map; i < ndesc; i++,
 800             p = efi_next_descriptor(p, efihdr->descriptor_size)) {
 801                 if (boothowto & RB_VERBOSE) {
 802                         if (p->md_type < nitems(types))
 803                                 type = types[p->md_type];
 804                         else
 805                                 type = "<INVALID>";
 806                         printf("%23s %012lx %12p %08lx ", type, p->md_phys,
 807                             p->md_virt, p->md_pages);
 808                         if (p->md_attr & EFI_MD_ATTR_UC)
 809                                 printf("UC ");
 810                         if (p->md_attr & EFI_MD_ATTR_WC)
 811                                 printf("WC ");
 812                         if (p->md_attr & EFI_MD_ATTR_WT)
 813                                 printf("WT ");
 814                         if (p->md_attr & EFI_MD_ATTR_WB)
 815                                 printf("WB ");
 816                         if (p->md_attr & EFI_MD_ATTR_UCE)
 817                                 printf("UCE ");
 818                         if (p->md_attr & EFI_MD_ATTR_WP)
 819                                 printf("WP ");
 820                         if (p->md_attr & EFI_MD_ATTR_RP)
 821                                 printf("RP ");
 822                         if (p->md_attr & EFI_MD_ATTR_XP)
 823                                 printf("XP ");
 824                         if (p->md_attr & EFI_MD_ATTR_NV)
 825                                 printf("NV ");
 826                         if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
 827                                 printf("MORE_RELIABLE ");
 828                         if (p->md_attr & EFI_MD_ATTR_RO)
 829                                 printf("RO ");
 830                         if (p->md_attr & EFI_MD_ATTR_RT)
 831                                 printf("RUNTIME");
 832                         printf("\n");
 833                 }
 834
 835                 switch (p->md_type) {
 836                 case EFI_MD_TYPE_CODE:
 837                 case EFI_MD_TYPE_DATA:
 838                 case EFI_MD_TYPE_BS_CODE:
 839                 case EFI_MD_TYPE_BS_DATA:
 840                 case EFI_MD_TYPE_FREE:
 841                         /*
 842                          * We're allowed to use any entry with these types.
 843                          */
 844                         break;
 845                 default:
 846                         continue;
 847                 }
 848
 849                 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE),
 850                     physmap, physmap_idxp))
 851                         break;
 852         }
 853 }
 854
 855 #ifdef FDT
 856 static void
 857 try_load_dtb(caddr_t kmdp)
 858 {
 859         vm_offset_t dtbp;
 860
 861         dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
 862         if (dtbp == (vm_offset_t)NULL) {
 863                 printf("ERROR loading DTB\n");
 864                 return;
 865         }
 866
 867         if (OF_install(OFW_FDT, 0) == FALSE)
 868                 panic("Cannot install FDT");
 869
 870         if (OF_init((void *)dtbp) != 0)
 871                 panic("OF_init failed with the found device tree");
 872 }
 873 #endif
 874
 875 static bool
 876 bus_probe(void)
 877 {
 878         bool has_acpi, has_fdt;
 879         char *order, *env;
 880
 881         has_acpi = has_fdt = false;
 882
 883 #ifdef FDT
 884         has_fdt = (OF_peer(0) != 0);
 885 #endif
 886 #ifdef DEV_ACPI
 887         has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0);
 888 #endif
 889
 890         env = kern_getenv("kern.cfg.order");
 891         if (env != NULL) {
 892                 order = env;
 893                 while (order != NULL) {
 894                         if (has_acpi &&
 895                             strncmp(order, "acpi", 4) == 0 &&
 896                             (order[4] == ',' || order[4] == '\0')) {
 897                                 arm64_bus_method = ARM64_BUS_ACPI;
 898                                 break;
 899                         }
 900                         if (has_fdt &&
 901                             strncmp(order, "fdt", 3) == 0 &&
 902                             (order[3] == ',' || order[3] == '\0')) {
 903                                 arm64_bus_method = ARM64_BUS_FDT;
 904                                 break;
 905                         }
 906                         order = strchr(order, ',');
 907                 }
 908                 freeenv(env);
 909
 910                 /* If we set the bus method it is valid */
 911                 if (arm64_bus_method != ARM64_BUS_NONE)
 912                         return (true);
 913         }
 914         /* If no order or an invalid order was set use the default */
 915         if (arm64_bus_method == ARM64_BUS_NONE) {
 916                 if (has_fdt)
 917                         arm64_bus_method = ARM64_BUS_FDT;
 918                 else if (has_acpi)
 919                         arm64_bus_method = ARM64_BUS_ACPI;
 920         }
 921
 922         /*
 923          * If no option was set the default is valid, otherwise we are
 924          * setting one to get cninit() working, then calling panic to tell
 925          * the user about the invalid bus setup.
 926          */
 927         return (env == NULL);
 928 }
 929
 930 static void
 931 cache_setup(void)
 932 {
 933         int dcache_line_shift, icache_line_shift, dczva_line_shift;
 934         uint32_t ctr_el0;
 935         uint32_t dczid_el0;
 936
 937         ctr_el0 = READ_SPECIALREG(ctr_el0);
 938
 939         /* Read the log2 words in each D cache line */
 940         dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
 941         /* Get the D cache line size */
 942         dcache_line_size = sizeof(int) << dcache_line_shift;
 943
 944         /* And the same for the I cache */
 945         icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
 946         icache_line_size = sizeof(int) << icache_line_shift;
 947
 948         idcache_line_size = MIN(dcache_line_size, icache_line_size);
 949
 950         dczid_el0 = READ_SPECIALREG(dczid_el0);
 951
 952         /* Check if dc zva is not prohibited */
 953         if (dczid_el0 & DCZID_DZP)
 954                 dczva_line_size = 0;
 955         else {
 956                 /* Same as with above calculations */
 957                 dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
 958                 dczva_line_size = sizeof(int) << dczva_line_shift;
 959
 960                 /* Change pagezero function */
 961                 pagezero = pagezero_cache;
 962         }
 963 }
 964
 965 void
 966 initarm(struct arm64_bootparams *abp)
 967 {
 968         struct efi_map_header *efihdr;
 969         struct pcpu *pcpup;
 970 #ifdef FDT
 971         struct mem_region mem_regions[FDT_MEM_REGIONS];
 972         int mem_regions_sz;
 973 #endif
 974         vm_offset_t lastaddr;
 975         caddr_t kmdp;
 976         vm_paddr_t mem_len;
 977         bool valid;
 978         int i;
 979
 980         /* Set the module data location */
 981         preload_metadata = (caddr_t)(uintptr_t)(abp->modulep);
 982
 983         /* Find the kernel address */
 984         kmdp = preload_search_by_type("elf kernel");
 985         if (kmdp == NULL)
 986                 kmdp = preload_search_by_type("elf64 kernel");
 987
 988         boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
 989         init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0);
 990
 991 #ifdef FDT
 992         try_load_dtb(kmdp);
 993 #endif
 994
 995         efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);
 996
 997         /* Find the address to start allocating from */
 998         lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
 999
1000         /* Load the physical memory ranges */
1001         physmap_idx = 0;
1002         efihdr = (struct efi_map_header *)preload_search_info(kmdp,
1003             MODINFO_METADATA | MODINFOMD_EFI_MAP);
1004         if (efihdr != NULL)
1005                 add_efi_map_entries(efihdr, physmap, &physmap_idx);
1006 #ifdef FDT
1007         else {
1008                 /* Grab physical memory regions information from device tree. */
1009                 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,
1010                     NULL) != 0)
1011                         panic("Cannot get physical memory regions");
1012                 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap,
1013                     &physmap_idx);
1014         }
1015 #endif
1016
1017         /* Print the memory map */
1018         mem_len = 0;
1019         for (i = 0; i < physmap_idx; i += 2) {
1020                 dump_avail[i] = physmap[i];
1021                 dump_avail[i + 1] = physmap[i + 1];
1022                 mem_len += physmap[i + 1] - physmap[i];
1023         }
1024         dump_avail[i] = 0;
1025         dump_avail[i + 1] = 0;
1026
1027         /* Set the pcpu data, this is needed by pmap_bootstrap */
1028         pcpup = &__pcpu[0];
1029         pcpu_init(pcpup, 0, sizeof(struct pcpu));
1030
1031         /*
1032          * Set the pcpu pointer with a backup in tpidr_el1 to be
1033          * loaded when entering the kernel from userland.
1034          */
1035         __asm __volatile(
1036             "mov x18, %0 \n"
1037             "msr tpidr_el1, %0" :: "r"(pcpup));
1038
1039         PCPU_SET(curthread, &thread0);
1040
1041         /* Do basic tuning, hz etc */
1042         init_param1();
1043
1044         cache_setup();
1045         pan_setup();
1046
1047         /* Bootstrap enough of pmap  to enter the kernel proper */
1048         pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt,
1049             KERNBASE - abp->kern_delta, lastaddr - KERNBASE);
1050
1051         devmap_bootstrap(0, NULL);
1052
1053         valid = bus_probe();
1054
1055         cninit();
1056
1057         if (!valid)
1058                 panic("Invalid bus configuration: %s",
1059                     kern_getenv("kern.cfg.order"));
1060
1061         init_proc0(abp->kern_stack);
1062         msgbufinit(msgbufp, msgbufsize);
1063         mutex_init();
1064         init_param2(physmem);
1065
1066         dbg_init();
1067         kdb_init();
1068         pan_enable();
1069
1070         early_boot = 0;
1071 }
1072
1073 void
1074 dbg_init(void)
1075 {
1076
1077         /* Clear OS lock */
1078         WRITE_SPECIALREG(OSLAR_EL1, 0);
1079
1080         /* This permits DDB to use debug registers for watchpoints. */
1081         dbg_monitor_init();
1082
1083         /* TODO: Eventually will need to initialize debug registers here. */
1084 }
1085
1086 #ifdef DDB
1087 #include <ddb/ddb.h>
1088
1089 DB_SHOW_COMMAND(specialregs, db_show_spregs)
1090 {
1091 #define PRINT_REG(reg)  \
1092     db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg))
1093
1094         PRINT_REG(actlr_el1);
1095         PRINT_REG(afsr0_el1);
1096         PRINT_REG(afsr1_el1);
1097         PRINT_REG(aidr_el1);
1098         PRINT_REG(amair_el1);
1099         PRINT_REG(ccsidr_el1);
1100         PRINT_REG(clidr_el1);
1101         PRINT_REG(contextidr_el1);
1102         PRINT_REG(cpacr_el1);
1103         PRINT_REG(csselr_el1);
1104         PRINT_REG(ctr_el0);
1105         PRINT_REG(currentel);
1106         PRINT_REG(daif);
1107         PRINT_REG(dczid_el0);
1108         PRINT_REG(elr_el1);
1109         PRINT_REG(esr_el1);
1110         PRINT_REG(far_el1);
1111 #if 0
1112         /* ARM64TODO: Enable VFP before reading floating-point registers */
1113         PRINT_REG(fpcr);
1114         PRINT_REG(fpsr);
1115 #endif
1116         PRINT_REG(id_aa64afr0_el1);
1117         PRINT_REG(id_aa64afr1_el1);
1118         PRINT_REG(id_aa64dfr0_el1);
1119         PRINT_REG(id_aa64dfr1_el1);
1120         PRINT_REG(id_aa64isar0_el1);
1121         PRINT_REG(id_aa64isar1_el1);
1122         PRINT_REG(id_aa64pfr0_el1);
1123         PRINT_REG(id_aa64pfr1_el1);
1124         PRINT_REG(id_afr0_el1);
1125         PRINT_REG(id_dfr0_el1);
1126         PRINT_REG(id_isar0_el1);
1127         PRINT_REG(id_isar1_el1);
1128         PRINT_REG(id_isar2_el1);
1129         PRINT_REG(id_isar3_el1);
1130         PRINT_REG(id_isar4_el1);
1131         PRINT_REG(id_isar5_el1);
1132         PRINT_REG(id_mmfr0_el1);
1133         PRINT_REG(id_mmfr1_el1);
1134         PRINT_REG(id_mmfr2_el1);
1135         PRINT_REG(id_mmfr3_el1);
1136 #if 0
1137         /* Missing from llvm */
1138         PRINT_REG(id_mmfr4_el1);
1139 #endif
1140         PRINT_REG(id_pfr0_el1);
1141         PRINT_REG(id_pfr1_el1);
1142         PRINT_REG(isr_el1);
1143         PRINT_REG(mair_el1);
1144         PRINT_REG(midr_el1);
1145         PRINT_REG(mpidr_el1);
1146         PRINT_REG(mvfr0_el1);
1147         PRINT_REG(mvfr1_el1);
1148         PRINT_REG(mvfr2_el1);
1149         PRINT_REG(revidr_el1);
1150         PRINT_REG(sctlr_el1);
1151         PRINT_REG(sp_el0);
1152         PRINT_REG(spsel);
1153         PRINT_REG(spsr_el1);
1154         PRINT_REG(tcr_el1);
1155         PRINT_REG(tpidr_el0);
1156         PRINT_REG(tpidr_el1);
1157         PRINT_REG(tpidrro_el0);
1158         PRINT_REG(ttbr0_el1);
1159         PRINT_REG(ttbr1_el1);
1160         PRINT_REG(vbar_el1);
1161 #undef PRINT_REG
1162 }
1163
1164 DB_SHOW_COMMAND(vtop, db_show_vtop)
1165 {
1166         uint64_t phys;
1167
1168         if (have_addr) {
1169                 phys = arm64_address_translate_s1e1r(addr);
1170                 db_printf("EL1 physical address reg (read):  0x%016lx\n", phys);
1171                 phys = arm64_address_translate_s1e1w(addr);
1172                 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys);
1173                 phys = arm64_address_translate_s1e0r(addr);
1174                 db_printf("EL0 physical address reg (read):  0x%016lx\n", phys);
1175                 phys = arm64_address_translate_s1e0w(addr);
1176                 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys);
1177         } else
1178                 db_printf("show vtop <virt_addr>\n");
1179 }
1180 #endif