sys/arm64/arm64/machdep.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  */
  27
  28 #include "opt_acpi.h"
  29 #include "opt_platform.h"
  30 #include "opt_ddb.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/buf.h>
  38 #include <sys/bus.h>
  39 #include <sys/cons.h>
  40 #include <sys/cpu.h>
  41 #include <sys/devmap.h>
  42 #include <sys/efi.h>
  43 #include <sys/exec.h>
  44 #include <sys/imgact.h>
  45 #include <sys/kdb.h>
  46 #include <sys/kernel.h>
  47 #include <sys/limits.h>
  48 #include <sys/linker.h>
  49 #include <sys/msgbuf.h>
  50 #include <sys/pcpu.h>
  51 #include <sys/proc.h>
  52 #include <sys/ptrace.h>
  53 #include <sys/reboot.h>
  54 #include <sys/rwlock.h>
  55 #include <sys/sched.h>
  56 #include <sys/signalvar.h>
  57 #include <sys/syscallsubr.h>
  58 #include <sys/sysent.h>
  59 #include <sys/sysproto.h>
  60 #include <sys/ucontext.h>
  61 #include <sys/vdso.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/vm_kern.h>
  65 #include <vm/vm_object.h>
  66 #include <vm/vm_page.h>
  67 #include <vm/pmap.h>
  68 #include <vm/vm_map.h>
  69 #include <vm/vm_pager.h>
  70
  71 #include <machine/armreg.h>
  72 #include <machine/cpu.h>
  73 #include <machine/debug_monitor.h>
  74 #include <machine/kdb.h>
  75 #include <machine/machdep.h>
  76 #include <machine/metadata.h>
  77 #include <machine/md_var.h>
  78 #include <machine/pcb.h>
  79 #include <machine/reg.h>
  80 #include <machine/undefined.h>
  81 #include <machine/vmparam.h>
  82
  83 #ifdef VFP
  84 #include <machine/vfp.h>
  85 #endif
  86
  87 #ifdef DEV_ACPI
  88 #include <contrib/dev/acpica/include/acpi.h>
  89 #include <machine/acpica_machdep.h>
  90 #endif
  91
  92 #ifdef FDT
  93 #include <dev/fdt/fdt_common.h>
  94 #include <dev/ofw/openfirm.h>
  95 #endif
  96
  97
  98 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE;
  99
 100 struct pcpu __pcpu[MAXCPU];
 101
 102 static struct trapframe proc0_tf;
 103
 104 vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2];
 105 vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2];
 106
 107 int early_boot = 1;
 108 int cold = 1;
 109 long realmem = 0;
 110 long Maxmem = 0;
 111
 112 #define PHYSMAP_SIZE    (2 * (VM_PHYSSEG_MAX - 1))
 113 vm_paddr_t physmap[PHYSMAP_SIZE];
 114 u_int physmap_idx;
 115
 116 struct kva_md_info kmi;
 117
 118 int64_t dcache_line_size;       /* The minimum D cache line size */
 119 int64_t icache_line_size;       /* The minimum I cache line size */
 120 int64_t idcache_line_size;      /* The minimum cache line size */
 121 int64_t dczva_line_size;        /* The size of cache line the dc zva zeroes */
 122 int has_pan;
 123
 124 /*
 125  * Physical address of the EFI System Table. Stashed from the metadata hints
 126  * passed into the kernel and used by the EFI code to call runtime services.
 127  */
 128 vm_paddr_t efi_systbl_phys;
 129
 130 /* pagezero_* implementations are provided in support.S */
 131 void pagezero_simple(void *);
 132 void pagezero_cache(void *);
 133
 134 /* pagezero_simple is default pagezero */
 135 void (*pagezero)(void *p) = pagezero_simple;
 136
 137 static void
 138 pan_setup(void)
 139 {
 140         uint64_t id_aa64mfr1;
 141
 142         id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
 143         if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE)
 144                 has_pan = 1;
 145 }
 146
 147 void
 148 pan_enable(void)
 149 {
 150
 151         /*
 152          * The LLVM integrated assembler doesn't understand the PAN
 153          * PSTATE field. Because of this we need to manually create
 154          * the instruction in an asm block. This is equivalent to:
 155          * msr pan, #1
 156          *
 157          * This sets the PAN bit, stopping the kernel from accessing
 158          * memory when userspace can also access it unless the kernel
 159          * uses the userspace load/store instructions.
 160          */
 161         if (has_pan) {
 162                 WRITE_SPECIALREG(sctlr_el1,
 163                     READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN);
 164                 __asm __volatile(".inst 0xd500409f | (0x1 << 8)");
 165         }
 166 }
 167
 168 static void
 169 cpu_startup(void *dummy)
 170 {
 171
 172         undef_init();
 173         identify_cpu();
 174
 175         vm_ksubmap_init(&kmi);
 176         bufinit();
 177         vm_pager_bufferinit();
 178 }
 179
 180 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 181
 182 int
 183 cpu_idle_wakeup(int cpu)
 184 {
 185
 186         return (0);
 187 }
 188
 189 int
 190 fill_regs(struct thread *td, struct reg *regs)
 191 {
 192         struct trapframe *frame;
 193
 194         frame = td->td_frame;
 195         regs->sp = frame->tf_sp;
 196         regs->lr = frame->tf_lr;
 197         regs->elr = frame->tf_elr;
 198         regs->spsr = frame->tf_spsr;
 199
 200         memcpy(regs->x, frame->tf_x, sizeof(regs->x));
 201
 202         return (0);
 203 }
 204
 205 int
 206 set_regs(struct thread *td, struct reg *regs)
 207 {
 208         struct trapframe *frame;
 209
 210         frame = td->td_frame;
 211         frame->tf_sp = regs->sp;
 212         frame->tf_lr = regs->lr;
 213         frame->tf_elr = regs->elr;
 214         frame->tf_spsr &= ~PSR_FLAGS;
 215         frame->tf_spsr |= regs->spsr & PSR_FLAGS;
 216
 217         memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x));
 218
 219         return (0);
 220 }
 221
 222 int
 223 fill_fpregs(struct thread *td, struct fpreg *regs)
 224 {
 225 #ifdef VFP
 226         struct pcb *pcb;
 227
 228         pcb = td->td_pcb;
 229         if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 230                 /*
 231                  * If we have just been running VFP instructions we will
 232                  * need to save the state to memcpy it below.
 233                  */
 234                 if (td == curthread)
 235                         vfp_save_state(td, pcb);
 236
 237                 KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 238                     ("Called fill_fpregs while the kernel is using the VFP"));
 239                 memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs,
 240                     sizeof(regs->fp_q));
 241                 regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr;
 242                 regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr;
 243         } else
 244 #endif
 245                 memset(regs->fp_q, 0, sizeof(regs->fp_q));
 246         return (0);
 247 }
 248
 249 int
 250 set_fpregs(struct thread *td, struct fpreg *regs)
 251 {
 252 #ifdef VFP
 253         struct pcb *pcb;
 254
 255         pcb = td->td_pcb;
 256         KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
 257             ("Called set_fpregs while the kernel is using the VFP"));
 258         memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q));
 259         pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr;
 260         pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr;
 261 #endif
 262         return (0);
 263 }
 264
 265 int
 266 fill_dbregs(struct thread *td, struct dbreg *regs)
 267 {
 268
 269         printf("ARM64TODO: fill_dbregs");
 270         return (EDOOFUS);
 271 }
 272
 273 int
 274 set_dbregs(struct thread *td, struct dbreg *regs)
 275 {
 276
 277         printf("ARM64TODO: set_dbregs");
 278         return (EDOOFUS);
 279 }
 280
 281 int
 282 ptrace_set_pc(struct thread *td, u_long addr)
 283 {
 284
 285         printf("ARM64TODO: ptrace_set_pc");
 286         return (EDOOFUS);
 287 }
 288
 289 int
 290 ptrace_single_step(struct thread *td)
 291 {
 292
 293         td->td_frame->tf_spsr |= PSR_SS;
 294         td->td_pcb->pcb_flags |= PCB_SINGLE_STEP;
 295         return (0);
 296 }
 297
 298 int
 299 ptrace_clear_single_step(struct thread *td)
 300 {
 301
 302         td->td_frame->tf_spsr &= ~PSR_SS;
 303         td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
 304         return (0);
 305 }
 306
 307 void
 308 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
 309 {
 310         struct trapframe *tf = td->td_frame;
 311
 312         memset(tf, 0, sizeof(struct trapframe));
 313
 314         tf->tf_x[0] = stack;
 315         tf->tf_sp = STACKALIGN(stack);
 316         tf->tf_lr = imgp->entry_addr;
 317         tf->tf_elr = imgp->entry_addr;
 318 }
 319
 320 /* Sanity check these are the same size, they will be memcpy'd to and fro */
 321 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 322     sizeof((struct gpregs *)0)->gp_x);
 323 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
 324     sizeof((struct reg *)0)->x);
 325
 326 int
 327 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret)
 328 {
 329         struct trapframe *tf = td->td_frame;
 330
 331         if (clear_ret & GET_MC_CLEAR_RET) {
 332                 mcp->mc_gpregs.gp_x[0] = 0;
 333                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C;
 334         } else {
 335                 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0];
 336                 mcp->mc_gpregs.gp_spsr = tf->tf_spsr;
 337         }
 338
 339         memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1],
 340             sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1));
 341
 342         mcp->mc_gpregs.gp_sp = tf->tf_sp;
 343         mcp->mc_gpregs.gp_lr = tf->tf_lr;
 344         mcp->mc_gpregs.gp_elr = tf->tf_elr;
 345
 346         return (0);
 347 }
 348
 349 int
 350 set_mcontext(struct thread *td, mcontext_t *mcp)
 351 {
 352         struct trapframe *tf = td->td_frame;
 353         uint32_t spsr;
 354
 355         spsr = mcp->mc_gpregs.gp_spsr;
 356         if ((spsr & PSR_M_MASK) != PSR_M_EL0t ||
 357             (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0)
 358                 return (EINVAL);
 359
 360         memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x));
 361
 362         tf->tf_sp = mcp->mc_gpregs.gp_sp;
 363         tf->tf_lr = mcp->mc_gpregs.gp_lr;
 364         tf->tf_elr = mcp->mc_gpregs.gp_elr;
 365         tf->tf_spsr = mcp->mc_gpregs.gp_spsr;
 366
 367         return (0);
 368 }
 369
 370 static void
 371 get_fpcontext(struct thread *td, mcontext_t *mcp)
 372 {
 373 #ifdef VFP
 374         struct pcb *curpcb;
 375
 376         critical_enter();
 377
 378         curpcb = curthread->td_pcb;
 379
 380         if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
 381                 /*
 382                  * If we have just been running VFP instructions we will
 383                  * need to save the state to memcpy it below.
 384                  */
 385                 vfp_save_state(td, curpcb);
 386
 387                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 388                     ("Called get_fpcontext while the kernel is using the VFP"));
 389                 KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
 390                     ("Non-userspace FPU flags set in get_fpcontext"));
 391                 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs,
 392                     sizeof(mcp->mc_fpregs));
 393                 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr;
 394                 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr;
 395                 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
 396                 mcp->mc_flags |= _MC_FP_VALID;
 397         }
 398
 399         critical_exit();
 400 #endif
 401 }
 402
 403 static void
 404 set_fpcontext(struct thread *td, mcontext_t *mcp)
 405 {
 406 #ifdef VFP
 407         struct pcb *curpcb;
 408
 409         critical_enter();
 410
 411         if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
 412                 curpcb = curthread->td_pcb;
 413
 414                 /*
 415                  * Discard any vfp state for the current thread, we
 416                  * are about to override it.
 417                  */
 418                 vfp_discard(td);
 419
 420                 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
 421                     ("Called set_fpcontext while the kernel is using the VFP"));
 422                 memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q,
 423                     sizeof(mcp->mc_fpregs));
 424                 curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr;
 425                 curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr;
 426                 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK;
 427         }
 428
 429         critical_exit();
 430 #endif
 431 }
 432
 433 void
 434 cpu_idle(int busy)
 435 {
 436
 437         spinlock_enter();
 438         if (!busy)
 439                 cpu_idleclock();
 440         if (!sched_runnable())
 441                 __asm __volatile(
 442                     "dsb sy \n"
 443                     "wfi    \n");
 444         if (!busy)
 445                 cpu_activeclock();
 446         spinlock_exit();
 447 }
 448
 449 void
 450 cpu_halt(void)
 451 {
 452
 453         /* We should have shutdown by now, if not enter a low power sleep */
 454         intr_disable();
 455         while (1) {
 456                 __asm __volatile("wfi");
 457         }
 458 }
 459
 460 /*
 461  * Flush the D-cache for non-DMA I/O so that the I-cache can
 462  * be made coherent later.
 463  */
 464 void
 465 cpu_flush_dcache(void *ptr, size_t len)
 466 {
 467
 468         /* ARM64TODO TBD */
 469 }
 470
 471 /* Get current clock frequency for the given CPU ID. */
 472 int
 473 cpu_est_clockrate(int cpu_id, uint64_t *rate)
 474 {
 475         struct pcpu *pc;
 476
 477         pc = pcpu_find(cpu_id);
 478         if (pc == NULL || rate == NULL)
 479                 return (EINVAL);
 480
 481         if (pc->pc_clock == 0)
 482                 return (EOPNOTSUPP);
 483
 484         *rate = pc->pc_clock;
 485         return (0);
 486 }
 487
 488 void
 489 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 490 {
 491
 492         pcpu->pc_acpi_id = 0xffffffff;
 493 }
 494
 495 void
 496 spinlock_enter(void)
 497 {
 498         struct thread *td;
 499         register_t daif;
 500
 501         td = curthread;
 502         if (td->td_md.md_spinlock_count == 0) {
 503                 daif = intr_disable();
 504                 td->td_md.md_spinlock_count = 1;
 505                 td->td_md.md_saved_daif = daif;
 506         } else
 507                 td->td_md.md_spinlock_count++;
 508         critical_enter();
 509 }
 510
 511 void
 512 spinlock_exit(void)
 513 {
 514         struct thread *td;
 515         register_t daif;
 516
 517         td = curthread;
 518         critical_exit();
 519         daif = td->td_md.md_saved_daif;
 520         td->td_md.md_spinlock_count--;
 521         if (td->td_md.md_spinlock_count == 0)
 522                 intr_restore(daif);
 523 }
 524
 525 #ifndef _SYS_SYSPROTO_H_
 526 struct sigreturn_args {
 527         ucontext_t *ucp;
 528 };
 529 #endif
 530
 531 int
 532 sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
 533 {
 534         ucontext_t uc;
 535         int error;
 536
 537         if (uap == NULL)
 538                 return (EFAULT);
 539         if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
 540                 return (EFAULT);
 541
 542         error = set_mcontext(td, &uc.uc_mcontext);
 543         if (error != 0)
 544                 return (error);
 545         set_fpcontext(td, &uc.uc_mcontext);
 546
 547         /* Restore signal mask. */
 548         kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
 549
 550         return (EJUSTRETURN);
 551 }
 552
 553 /*
 554  * Construct a PCB from a trapframe. This is called from kdb_trap() where
 555  * we want to start a backtrace from the function that caused us to enter
 556  * the debugger. We have the context in the trapframe, but base the trace
 557  * on the PCB. The PCB doesn't have to be perfect, as long as it contains
 558  * enough for a backtrace.
 559  */
 560 void
 561 makectx(struct trapframe *tf, struct pcb *pcb)
 562 {
 563         int i;
 564
 565         for (i = 0; i < PCB_LR; i++)
 566                 pcb->pcb_x[i] = tf->tf_x[i];
 567
 568         pcb->pcb_x[PCB_LR] = tf->tf_lr;
 569         pcb->pcb_pc = tf->tf_elr;
 570         pcb->pcb_sp = tf->tf_sp;
 571 }
 572
 573 void
 574 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 575 {
 576         struct thread *td;
 577         struct proc *p;
 578         struct trapframe *tf;
 579         struct sigframe *fp, frame;
 580         struct sigacts *psp;
 581         struct sysentvec *sysent;
 582         int code, onstack, sig;
 583
 584         td = curthread;
 585         p = td->td_proc;
 586         PROC_LOCK_ASSERT(p, MA_OWNED);
 587
 588         sig = ksi->ksi_signo;
 589         code = ksi->ksi_code;
 590         psp = p->p_sigacts;
 591         mtx_assert(&psp->ps_mtx, MA_OWNED);
 592
 593         tf = td->td_frame;
 594         onstack = sigonstack(tf->tf_sp);
 595
 596         CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
 597             catcher, sig);
 598
 599         /* Allocate and validate space for the signal handler context. */
 600         if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
 601             SIGISMEMBER(psp->ps_sigonstack, sig)) {
 602                 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp +
 603                     td->td_sigstk.ss_size);
 604 #if defined(COMPAT_43)
 605                 td->td_sigstk.ss_flags |= SS_ONSTACK;
 606 #endif
 607         } else {
 608                 fp = (struct sigframe *)td->td_frame->tf_sp;
 609         }
 610
 611         /* Make room, keeping the stack aligned */
 612         fp--;
 613         fp = (struct sigframe *)STACKALIGN(fp);
 614
 615         /* Fill in the frame to copy out */
 616         get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
 617         get_fpcontext(td, &frame.sf_uc.uc_mcontext);
 618         frame.sf_si = ksi->ksi_info;
 619         frame.sf_uc.uc_sigmask = *mask;
 620         frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ?
 621             ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE;
 622         frame.sf_uc.uc_stack = td->td_sigstk;
 623         mtx_unlock(&psp->ps_mtx);
 624         PROC_UNLOCK(td->td_proc);
 625
 626         /* Copy the sigframe out to the user's stack. */
 627         if (copyout(&frame, fp, sizeof(*fp)) != 0) {
 628                 /* Process has trashed its stack. Kill it. */
 629                 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
 630                 PROC_LOCK(p);
 631                 sigexit(td, SIGILL);
 632         }
 633
 634         tf->tf_x[0]= sig;
 635         tf->tf_x[1] = (register_t)&fp->sf_si;
 636         tf->tf_x[2] = (register_t)&fp->sf_uc;
 637
 638         tf->tf_elr = (register_t)catcher;
 639         tf->tf_sp = (register_t)fp;
 640         sysent = p->p_sysent;
 641         if (sysent->sv_sigcode_base != 0)
 642                 tf->tf_lr = (register_t)sysent->sv_sigcode_base;
 643         else
 644                 tf->tf_lr = (register_t)(sysent->sv_psstrings -
 645                     *(sysent->sv_szsigcode));
 646
 647         CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr,
 648             tf->tf_sp);
 649
 650         PROC_LOCK(p);
 651         mtx_lock(&psp->ps_mtx);
 652 }
 653
 654 static void
 655 init_proc0(vm_offset_t kstack)
 656 {
 657         struct pcpu *pcpup = &__pcpu[0];
 658
 659         proc_linkup0(&proc0, &thread0);
 660         thread0.td_kstack = kstack;
 661         thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1;
 662         thread0.td_pcb->pcb_fpflags = 0;
 663         thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate;
 664         thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
 665         thread0.td_frame = &proc0_tf;
 666         pcpup->pc_curpcb = thread0.td_pcb;
 667 }
 668
 669 typedef struct {
 670         uint32_t type;
 671         uint64_t phys_start;
 672         uint64_t virt_start;
 673         uint64_t num_pages;
 674         uint64_t attr;
 675 } EFI_MEMORY_DESCRIPTOR;
 676
 677 static int
 678 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
 679     u_int *physmap_idxp)
 680 {
 681         u_int i, insert_idx, _physmap_idx;
 682
 683         _physmap_idx = *physmap_idxp;
 684
 685         if (length == 0)
 686                 return (1);
 687
 688         /*
 689          * Find insertion point while checking for overlap.  Start off by
 690          * assuming the new entry will be added to the end.
 691          */
 692         insert_idx = _physmap_idx;
 693         for (i = 0; i <= _physmap_idx; i += 2) {
 694                 if (base < physmap[i + 1]) {
 695                         if (base + length <= physmap[i]) {
 696                                 insert_idx = i;
 697                                 break;
 698                         }
 699                         if (boothowto & RB_VERBOSE)
 700                                 printf(
 701                     "Overlapping memory regions, ignoring second region\n");
 702                         return (1);
 703                 }
 704         }
 705
 706         /* See if we can prepend to the next entry. */
 707         if (insert_idx <= _physmap_idx &&
 708             base + length == physmap[insert_idx]) {
 709                 physmap[insert_idx] = base;
 710                 return (1);
 711         }
 712
 713         /* See if we can append to the previous entry. */
 714         if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
 715                 physmap[insert_idx - 1] += length;
 716                 return (1);
 717         }
 718
 719         _physmap_idx += 2;
 720         *physmap_idxp = _physmap_idx;
 721         if (_physmap_idx == PHYSMAP_SIZE) {
 722                 printf(
 723                 "Too many segments in the physical address map, giving up\n");
 724                 return (0);
 725         }
 726
 727         /*
 728          * Move the last 'N' entries down to make room for the new
 729          * entry if needed.
 730          */
 731         for (i = _physmap_idx; i > insert_idx; i -= 2) {
 732                 physmap[i] = physmap[i - 2];
 733                 physmap[i + 1] = physmap[i - 1];
 734         }
 735
 736         /* Insert the new entry. */
 737         physmap[insert_idx] = base;
 738         physmap[insert_idx + 1] = base + length;
 739         return (1);
 740 }
 741
 742 #ifdef FDT
 743 static void
 744 add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap,
 745     u_int *physmap_idxp)
 746 {
 747
 748         for (int i = 0; i < mrcnt; i++) {
 749                 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap,
 750                     physmap_idxp))
 751                         break;
 752         }
 753 }
 754 #endif
 755
 756 static void
 757 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
 758     u_int *physmap_idxp)
 759 {
 760         struct efi_md *map, *p;
 761         const char *type;
 762         size_t efisz;
 763         int ndesc, i;
 764
 765         static const char *types[] = {
 766                 "Reserved",
 767                 "LoaderCode",
 768                 "LoaderData",
 769                 "BootServicesCode",
 770                 "BootServicesData",
 771                 "RuntimeServicesCode",
 772                 "RuntimeServicesData",
 773                 "ConventionalMemory",
 774                 "UnusableMemory",
 775                 "ACPIReclaimMemory",
 776                 "ACPIMemoryNVS",
 777                 "MemoryMappedIO",
 778                 "MemoryMappedIOPortSpace",
 779                 "PalCode",
 780                 "PersistentMemory"
 781         };
 782
 783         /*
 784          * Memory map data provided by UEFI via the GetMemoryMap
 785          * Boot Services API.
 786          */
 787         efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
 788         map = (struct efi_md *)((uint8_t *)efihdr + efisz);
 789
 790         if (efihdr->descriptor_size == 0)
 791                 return;
 792         ndesc = efihdr->memory_size / efihdr->descriptor_size;
 793
 794         if (boothowto & RB_VERBOSE)
 795                 printf("%23s %12s %12s %8s %4s\n",
 796                     "Type", "Physical", "Virtual", "#Pages", "Attr");
 797
 798         for (i = 0, p = map; i < ndesc; i++,
 799             p = efi_next_descriptor(p, efihdr->descriptor_size)) {
 800                 if (boothowto & RB_VERBOSE) {
 801                         if (p->md_type < nitems(types))
 802                                 type = types[p->md_type];
 803                         else
 804                                 type = "<INVALID>";
 805                         printf("%23s %012lx %12p %08lx ", type, p->md_phys,
 806                             p->md_virt, p->md_pages);
 807                         if (p->md_attr & EFI_MD_ATTR_UC)
 808                                 printf("UC ");
 809                         if (p->md_attr & EFI_MD_ATTR_WC)
 810                                 printf("WC ");
 811                         if (p->md_attr & EFI_MD_ATTR_WT)
 812                                 printf("WT ");
 813                         if (p->md_attr & EFI_MD_ATTR_WB)
 814                                 printf("WB ");
 815                         if (p->md_attr & EFI_MD_ATTR_UCE)
 816                                 printf("UCE ");
 817                         if (p->md_attr & EFI_MD_ATTR_WP)
 818                                 printf("WP ");
 819                         if (p->md_attr & EFI_MD_ATTR_RP)
 820                                 printf("RP ");
 821                         if (p->md_attr & EFI_MD_ATTR_XP)
 822                                 printf("XP ");
 823                         if (p->md_attr & EFI_MD_ATTR_NV)
 824                                 printf("NV ");
 825                         if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
 826                                 printf("MORE_RELIABLE ");
 827                         if (p->md_attr & EFI_MD_ATTR_RO)
 828                                 printf("RO ");
 829                         if (p->md_attr & EFI_MD_ATTR_RT)
 830                                 printf("RUNTIME");
 831                         printf("\n");
 832                 }
 833
 834                 switch (p->md_type) {
 835                 case EFI_MD_TYPE_CODE:
 836                 case EFI_MD_TYPE_DATA:
 837                 case EFI_MD_TYPE_BS_CODE:
 838                 case EFI_MD_TYPE_BS_DATA:
 839                 case EFI_MD_TYPE_FREE:
 840                         /*
 841                          * We're allowed to use any entry with these types.
 842                          */
 843                         break;
 844                 default:
 845                         continue;
 846                 }
 847
 848                 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE),
 849                     physmap, physmap_idxp))
 850                         break;
 851         }
 852 }
 853
 854 #ifdef FDT
 855 static void
 856 try_load_dtb(caddr_t kmdp)
 857 {
 858         vm_offset_t dtbp;
 859
 860         dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
 861         if (dtbp == (vm_offset_t)NULL) {
 862                 printf("ERROR loading DTB\n");
 863                 return;
 864         }
 865
 866         if (OF_install(OFW_FDT, 0) == FALSE)
 867                 panic("Cannot install FDT");
 868
 869         if (OF_init((void *)dtbp) != 0)
 870                 panic("OF_init failed with the found device tree");
 871 }
 872 #endif
 873
 874 static bool
 875 bus_probe(void)
 876 {
 877         bool has_acpi, has_fdt;
 878         char *order, *env;
 879
 880         has_acpi = has_fdt = false;
 881
 882 #ifdef FDT
 883         has_fdt = (OF_peer(0) != 0);
 884 #endif
 885 #ifdef DEV_ACPI
 886         has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0);
 887 #endif
 888
 889         env = kern_getenv("kern.cfg.order");
 890         if (env != NULL) {
 891                 order = env;
 892                 while (order != NULL) {
 893                         if (has_acpi &&
 894                             strncmp(order, "acpi", 4) == 0 &&
 895                             (order[4] == ',' || order[4] == '\0')) {
 896                                 arm64_bus_method = ARM64_BUS_ACPI;
 897                                 break;
 898                         }
 899                         if (has_fdt &&
 900                             strncmp(order, "fdt", 3) == 0 &&
 901                             (order[3] == ',' || order[3] == '\0')) {
 902                                 arm64_bus_method = ARM64_BUS_FDT;
 903                                 break;
 904                         }
 905                         order = strchr(order, ',');
 906                 }
 907                 freeenv(env);
 908
 909                 /* If we set the bus method it is valid */
 910                 if (arm64_bus_method != ARM64_BUS_NONE)
 911                         return (true);
 912         }
 913         /* If no order or an invalid order was set use the default */
 914         if (arm64_bus_method == ARM64_BUS_NONE) {
 915                 if (has_fdt)
 916                         arm64_bus_method = ARM64_BUS_FDT;
 917                 else if (has_acpi)
 918                         arm64_bus_method = ARM64_BUS_ACPI;
 919         }
 920
 921         /*
 922          * If no option was set the default is valid, otherwise we are
 923          * setting one to get cninit() working, then calling panic to tell
 924          * the user about the invalid bus setup.
 925          */
 926         return (env == NULL);
 927 }
 928
 929 static void
 930 cache_setup(void)
 931 {
 932         int dcache_line_shift, icache_line_shift, dczva_line_shift;
 933         uint32_t ctr_el0;
 934         uint32_t dczid_el0;
 935
 936         ctr_el0 = READ_SPECIALREG(ctr_el0);
 937
 938         /* Read the log2 words in each D cache line */
 939         dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
 940         /* Get the D cache line size */
 941         dcache_line_size = sizeof(int) << dcache_line_shift;
 942
 943         /* And the same for the I cache */
 944         icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
 945         icache_line_size = sizeof(int) << icache_line_shift;
 946
 947         idcache_line_size = MIN(dcache_line_size, icache_line_size);
 948
 949         dczid_el0 = READ_SPECIALREG(dczid_el0);
 950
 951         /* Check if dc zva is not prohibited */
 952         if (dczid_el0 & DCZID_DZP)
 953                 dczva_line_size = 0;
 954         else {
 955                 /* Same as with above calculations */
 956                 dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
 957                 dczva_line_size = sizeof(int) << dczva_line_shift;
 958
 959                 /* Change pagezero function */
 960                 pagezero = pagezero_cache;
 961         }
 962 }
 963
 964 void
 965 initarm(struct arm64_bootparams *abp)
 966 {
 967         struct efi_map_header *efihdr;
 968         struct pcpu *pcpup;
 969 #ifdef FDT
 970         struct mem_region mem_regions[FDT_MEM_REGIONS];
 971         int mem_regions_sz;
 972 #endif
 973         vm_offset_t lastaddr;
 974         caddr_t kmdp;
 975         vm_paddr_t mem_len;
 976         bool valid;
 977         int i;
 978
 979         /* Set the module data location */
 980         preload_metadata = (caddr_t)(uintptr_t)(abp->modulep);
 981
 982         /* Find the kernel address */
 983         kmdp = preload_search_by_type("elf kernel");
 984         if (kmdp == NULL)
 985                 kmdp = preload_search_by_type("elf64 kernel");
 986
 987         boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
 988         init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0);
 989
 990 #ifdef FDT
 991         try_load_dtb(kmdp);
 992 #endif
 993
 994         efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);
 995
 996         /* Find the address to start allocating from */
 997         lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
 998
 999         /* Load the physical memory ranges */
1000         physmap_idx = 0;
1001         efihdr = (struct efi_map_header *)preload_search_info(kmdp,
1002             MODINFO_METADATA | MODINFOMD_EFI_MAP);
1003         if (efihdr != NULL)
1004                 add_efi_map_entries(efihdr, physmap, &physmap_idx);
1005 #ifdef FDT
1006         else {
1007                 /* Grab physical memory regions information from device tree. */
1008                 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,
1009                     NULL) != 0)
1010                         panic("Cannot get physical memory regions");
1011                 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap,
1012                     &physmap_idx);
1013         }
1014 #endif
1015
1016         /* Print the memory map */
1017         mem_len = 0;
1018         for (i = 0; i < physmap_idx; i += 2) {
1019                 dump_avail[i] = physmap[i];
1020                 dump_avail[i + 1] = physmap[i + 1];
1021                 mem_len += physmap[i + 1] - physmap[i];
1022         }
1023         dump_avail[i] = 0;
1024         dump_avail[i + 1] = 0;
1025
1026         /* Set the pcpu data, this is needed by pmap_bootstrap */
1027         pcpup = &__pcpu[0];
1028         pcpu_init(pcpup, 0, sizeof(struct pcpu));
1029
1030         /*
1031          * Set the pcpu pointer with a backup in tpidr_el1 to be
1032          * loaded when entering the kernel from userland.
1033          */
1034         __asm __volatile(
1035             "mov x18, %0 \n"
1036             "msr tpidr_el1, %0" :: "r"(pcpup));
1037
1038         PCPU_SET(curthread, &thread0);
1039
1040         /* Do basic tuning, hz etc */
1041         init_param1();
1042
1043         cache_setup();
1044         pan_setup();
1045
1046         /* Bootstrap enough of pmap  to enter the kernel proper */
1047         pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt,
1048             KERNBASE - abp->kern_delta, lastaddr - KERNBASE);
1049
1050         devmap_bootstrap(0, NULL);
1051
1052         valid = bus_probe();
1053
1054         cninit();
1055
1056         if (!valid)
1057                 panic("Invalid bus configuration: %s",
1058                     kern_getenv("kern.cfg.order"));
1059
1060         init_proc0(abp->kern_stack);
1061         msgbufinit(msgbufp, msgbufsize);
1062         mutex_init();
1063         init_param2(physmem);
1064
1065         dbg_init();
1066         kdb_init();
1067         pan_enable();
1068
1069         early_boot = 0;
1070 }
1071
1072 void
1073 dbg_init(void)
1074 {
1075
1076         /* Clear OS lock */
1077         WRITE_SPECIALREG(OSLAR_EL1, 0);
1078
1079         /* This permits DDB to use debug registers for watchpoints. */
1080         dbg_monitor_init();
1081
1082         /* TODO: Eventually will need to initialize debug registers here. */
1083 }
1084
1085 #ifdef DDB
1086 #include <ddb/ddb.h>
1087
1088 DB_SHOW_COMMAND(specialregs, db_show_spregs)
1089 {
1090 #define PRINT_REG(reg)  \
1091     db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg))
1092
1093         PRINT_REG(actlr_el1);
1094         PRINT_REG(afsr0_el1);
1095         PRINT_REG(afsr1_el1);
1096         PRINT_REG(aidr_el1);
1097         PRINT_REG(amair_el1);
1098         PRINT_REG(ccsidr_el1);
1099         PRINT_REG(clidr_el1);
1100         PRINT_REG(contextidr_el1);
1101         PRINT_REG(cpacr_el1);
1102         PRINT_REG(csselr_el1);
1103         PRINT_REG(ctr_el0);
1104         PRINT_REG(currentel);
1105         PRINT_REG(daif);
1106         PRINT_REG(dczid_el0);
1107         PRINT_REG(elr_el1);
1108         PRINT_REG(esr_el1);
1109         PRINT_REG(far_el1);
1110 #if 0
1111         /* ARM64TODO: Enable VFP before reading floating-point registers */
1112         PRINT_REG(fpcr);
1113         PRINT_REG(fpsr);
1114 #endif
1115         PRINT_REG(id_aa64afr0_el1);
1116         PRINT_REG(id_aa64afr1_el1);
1117         PRINT_REG(id_aa64dfr0_el1);
1118         PRINT_REG(id_aa64dfr1_el1);
1119         PRINT_REG(id_aa64isar0_el1);
1120         PRINT_REG(id_aa64isar1_el1);
1121         PRINT_REG(id_aa64pfr0_el1);
1122         PRINT_REG(id_aa64pfr1_el1);
1123         PRINT_REG(id_afr0_el1);
1124         PRINT_REG(id_dfr0_el1);
1125         PRINT_REG(id_isar0_el1);
1126         PRINT_REG(id_isar1_el1);
1127         PRINT_REG(id_isar2_el1);
1128         PRINT_REG(id_isar3_el1);
1129         PRINT_REG(id_isar4_el1);
1130         PRINT_REG(id_isar5_el1);
1131         PRINT_REG(id_mmfr0_el1);
1132         PRINT_REG(id_mmfr1_el1);
1133         PRINT_REG(id_mmfr2_el1);
1134         PRINT_REG(id_mmfr3_el1);
1135 #if 0
1136         /* Missing from llvm */
1137         PRINT_REG(id_mmfr4_el1);
1138 #endif
1139         PRINT_REG(id_pfr0_el1);
1140         PRINT_REG(id_pfr1_el1);
1141         PRINT_REG(isr_el1);
1142         PRINT_REG(mair_el1);
1143         PRINT_REG(midr_el1);
1144         PRINT_REG(mpidr_el1);
1145         PRINT_REG(mvfr0_el1);
1146         PRINT_REG(mvfr1_el1);
1147         PRINT_REG(mvfr2_el1);
1148         PRINT_REG(revidr_el1);
1149         PRINT_REG(sctlr_el1);
1150         PRINT_REG(sp_el0);
1151         PRINT_REG(spsel);
1152         PRINT_REG(spsr_el1);
1153         PRINT_REG(tcr_el1);
1154         PRINT_REG(tpidr_el0);
1155         PRINT_REG(tpidr_el1);
1156         PRINT_REG(tpidrro_el0);
1157         PRINT_REG(ttbr0_el1);
1158         PRINT_REG(ttbr1_el1);
1159         PRINT_REG(vbar_el1);
1160 #undef PRINT_REG
1161 }
1162
1163 DB_SHOW_COMMAND(vtop, db_show_vtop)
1164 {
1165         uint64_t phys;
1166
1167         if (have_addr) {
1168                 phys = arm64_address_translate_s1e1r(addr);
1169                 db_printf("EL1 physical address reg (read):  0x%016lx\n", phys);
1170                 phys = arm64_address_translate_s1e1w(addr);
1171                 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys);
1172                 phys = arm64_address_translate_s1e0r(addr);
1173                 db_printf("EL0 physical address reg (read):  0x%016lx\n", phys);
1174                 phys = arm64_address_translate_s1e0w(addr);
1175                 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys);
1176         } else
1177                 db_printf("show vtop <virt_addr>\n");
1178 }
1179 #endif