2 * Copyright (c) 2014 Andrew Turner
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include "opt_platform.h"
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
41 #include <sys/imgact.h>
43 #include <sys/kernel.h>
44 #include <sys/limits.h>
45 #include <sys/linker.h>
46 #include <sys/msgbuf.h>
49 #include <sys/ptrace.h>
50 #include <sys/reboot.h>
51 #include <sys/rwlock.h>
52 #include <sys/sched.h>
53 #include <sys/signalvar.h>
54 #include <sys/syscallsubr.h>
55 #include <sys/sysent.h>
56 #include <sys/sysproto.h>
57 #include <sys/ucontext.h>
60 #include <vm/vm_kern.h>
61 #include <vm/vm_object.h>
62 #include <vm/vm_page.h>
64 #include <vm/vm_map.h>
65 #include <vm/vm_pager.h>
67 #include <machine/armreg.h>
68 #include <machine/cpu.h>
69 #include <machine/debug_monitor.h>
70 #include <machine/kdb.h>
71 #include <machine/devmap.h>
72 #include <machine/machdep.h>
73 #include <machine/metadata.h>
74 #include <machine/pcb.h>
75 #include <machine/reg.h>
76 #include <machine/vmparam.h>
79 #include <machine/vfp.h>
83 #include <dev/fdt/fdt_common.h>
84 #include <dev/ofw/openfirm.h>
87 struct pcpu __pcpu[MAXCPU];
89 static struct trapframe proc0_tf;
91 vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2];
92 vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2];
99 #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
100 vm_paddr_t physmap[PHYSMAP_SIZE];
103 struct kva_md_info kmi;
105 int64_t dcache_line_size; /* The minimum D cache line size */
106 int64_t icache_line_size; /* The minimum I cache line size */
107 int64_t idcache_line_size; /* The minimum cache line size */
110 cpu_startup(void *dummy)
115 vm_ksubmap_init(&kmi);
117 vm_pager_bufferinit();
120 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
123 bzero(void *buf, size_t len)
133 fill_regs(struct thread *td, struct reg *regs)
135 struct trapframe *frame;
137 frame = td->td_frame;
138 regs->sp = frame->tf_sp;
139 regs->lr = frame->tf_lr;
140 regs->elr = frame->tf_elr;
141 regs->spsr = frame->tf_spsr;
143 memcpy(regs->x, frame->tf_x, sizeof(regs->x));
149 set_regs(struct thread *td, struct reg *regs)
151 struct trapframe *frame;
153 frame = td->td_frame;
154 frame->tf_sp = regs->sp;
155 frame->tf_lr = regs->lr;
156 frame->tf_elr = regs->elr;
157 frame->tf_spsr = regs->spsr;
159 memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x));
165 fill_fpregs(struct thread *td, struct fpreg *regs)
171 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
173 * If we have just been running VFP instructions we will
174 * need to save the state to memcpy it below.
178 memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q));
179 regs->fp_cr = pcb->pcb_fpcr;
180 regs->fp_sr = pcb->pcb_fpsr;
183 memset(regs->fp_q, 0, sizeof(regs->fp_q));
188 set_fpregs(struct thread *td, struct fpreg *regs)
194 memcpy(pcb->pcb_vfp, regs->fp_q, sizeof(regs->fp_q));
195 pcb->pcb_fpcr = regs->fp_cr;
196 pcb->pcb_fpsr = regs->fp_sr;
202 fill_dbregs(struct thread *td, struct dbreg *regs)
205 panic("fill_dbregs");
209 set_dbregs(struct thread *td, struct dbreg *regs)
216 ptrace_set_pc(struct thread *td, u_long addr)
219 panic("ptrace_set_pc");
224 ptrace_single_step(struct thread *td)
232 ptrace_clear_single_step(struct thread *td)
240 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
242 struct trapframe *tf = td->td_frame;
244 memset(tf, 0, sizeof(struct trapframe));
247 tf->tf_lr = imgp->entry_addr;
248 tf->tf_elr = imgp->entry_addr;
251 /* Sanity check these are the same size, they will be memcpy'd to and fro */
252 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
253 sizeof((struct gpregs *)0)->gp_x);
254 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
255 sizeof((struct reg *)0)->x);
258 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret)
260 struct trapframe *tf = td->td_frame;
262 if (clear_ret & GET_MC_CLEAR_RET)
263 mcp->mc_gpregs.gp_x[0] = 0;
265 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0];
267 memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1],
268 sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1));
270 mcp->mc_gpregs.gp_sp = tf->tf_sp;
271 mcp->mc_gpregs.gp_lr = tf->tf_lr;
272 mcp->mc_gpregs.gp_elr = tf->tf_elr;
273 mcp->mc_gpregs.gp_spsr = tf->tf_spsr;
279 set_mcontext(struct thread *td, mcontext_t *mcp)
281 struct trapframe *tf = td->td_frame;
283 memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x));
285 tf->tf_sp = mcp->mc_gpregs.gp_sp;
286 tf->tf_lr = mcp->mc_gpregs.gp_lr;
287 tf->tf_elr = mcp->mc_gpregs.gp_elr;
288 tf->tf_spsr = mcp->mc_gpregs.gp_spsr;
294 get_fpcontext(struct thread *td, mcontext_t *mcp)
301 curpcb = curthread->td_pcb;
303 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
305 * If we have just been running VFP instructions we will
306 * need to save the state to memcpy it below.
310 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_vfp,
311 sizeof(mcp->mc_fpregs));
312 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpcr;
313 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpsr;
314 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
315 mcp->mc_flags |= _MC_FP_VALID;
323 set_fpcontext(struct thread *td, mcontext_t *mcp)
330 if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
331 curpcb = curthread->td_pcb;
334 * Discard any vfp state for the current thread, we
335 * are about to override it.
339 memcpy(curpcb->pcb_vfp, mcp->mc_fpregs.fp_q,
340 sizeof(mcp->mc_fpregs));
341 curpcb->pcb_fpcr = mcp->mc_fpregs.fp_cr;
342 curpcb->pcb_fpsr = mcp->mc_fpregs.fp_sr;
343 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags;
357 if (!sched_runnable())
374 * Flush the D-cache for non-DMA I/O so that the I-cache can
375 * be made coherent later.
378 cpu_flush_dcache(void *ptr, size_t len)
384 /* Get current clock frequency for the given CPU ID. */
386 cpu_est_clockrate(int cpu_id, uint64_t *rate)
389 panic("cpu_est_clockrate");
393 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
396 pcpu->pc_acpi_id = 0xffffffff;
406 if (td->td_md.md_spinlock_count == 0) {
407 daif = intr_disable();
408 td->td_md.md_spinlock_count = 1;
409 td->td_md.md_saved_daif = daif;
411 td->td_md.md_spinlock_count++;
423 daif = td->td_md.md_saved_daif;
424 td->td_md.md_spinlock_count--;
425 if (td->td_md.md_spinlock_count == 0)
429 #ifndef _SYS_SYSPROTO_H_
430 struct sigreturn_args {
436 sys_sigreturn(struct thread *td, struct sigreturn_args *uap)
443 if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
446 spsr = uc.uc_mcontext.mc_gpregs.gp_spsr;
447 if ((spsr & PSR_M_MASK) != PSR_M_EL0t ||
448 (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0)
451 set_mcontext(td, &uc.uc_mcontext);
452 set_fpcontext(td, &uc.uc_mcontext);
454 /* Restore signal mask. */
455 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
457 return (EJUSTRETURN);
461 * Construct a PCB from a trapframe. This is called from kdb_trap() where
462 * we want to start a backtrace from the function that caused us to enter
463 * the debugger. We have the context in the trapframe, but base the trace
464 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
465 * enough for a backtrace.
468 makectx(struct trapframe *tf, struct pcb *pcb)
472 for (i = 0; i < PCB_LR; i++)
473 pcb->pcb_x[i] = tf->tf_x[i];
475 pcb->pcb_x[PCB_LR] = tf->tf_lr;
476 pcb->pcb_pc = tf->tf_elr;
477 pcb->pcb_sp = tf->tf_sp;
481 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
485 struct trapframe *tf;
486 struct sigframe *fp, frame;
488 int code, onstack, sig;
492 PROC_LOCK_ASSERT(p, MA_OWNED);
494 sig = ksi->ksi_signo;
495 code = ksi->ksi_code;
497 mtx_assert(&psp->ps_mtx, MA_OWNED);
500 onstack = sigonstack(tf->tf_sp);
502 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
505 /* Allocate and validate space for the signal handler context. */
506 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
507 SIGISMEMBER(psp->ps_sigonstack, sig)) {
508 fp = (struct sigframe *)(td->td_sigstk.ss_sp +
509 td->td_sigstk.ss_size);
510 #if defined(COMPAT_43)
511 td->td_sigstk.ss_flags |= SS_ONSTACK;
514 fp = (struct sigframe *)td->td_frame->tf_sp;
517 /* Make room, keeping the stack aligned */
519 fp = (struct sigframe *)STACKALIGN(fp);
521 /* Fill in the frame to copy out */
522 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
523 get_fpcontext(td, &frame.sf_uc.uc_mcontext);
524 frame.sf_si = ksi->ksi_info;
525 frame.sf_uc.uc_sigmask = *mask;
526 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ?
527 ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE;
528 frame.sf_uc.uc_stack = td->td_sigstk;
529 mtx_unlock(&psp->ps_mtx);
530 PROC_UNLOCK(td->td_proc);
532 /* Copy the sigframe out to the user's stack. */
533 if (copyout(&frame, fp, sizeof(*fp)) != 0) {
534 /* Process has trashed its stack. Kill it. */
535 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
541 tf->tf_x[1] = (register_t)&fp->sf_si;
542 tf->tf_x[2] = (register_t)&fp->sf_uc;
544 tf->tf_elr = (register_t)catcher;
545 tf->tf_sp = (register_t)fp;
546 tf->tf_lr = (register_t)(PS_STRINGS - *(p->p_sysent->sv_szsigcode));
548 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr,
552 mtx_lock(&psp->ps_mtx);
556 init_proc0(vm_offset_t kstack)
558 struct pcpu *pcpup = &__pcpu[0];
560 proc_linkup0(&proc0, &thread0);
561 thread0.td_kstack = kstack;
562 thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1;
563 thread0.td_pcb->pcb_fpflags = 0;
564 thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
565 thread0.td_frame = &proc0_tf;
566 pcpup->pc_curpcb = thread0.td_pcb;
575 } EFI_MEMORY_DESCRIPTOR;
578 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
581 u_int i, insert_idx, _physmap_idx;
583 _physmap_idx = *physmap_idxp;
589 * Find insertion point while checking for overlap. Start off by
590 * assuming the new entry will be added to the end.
592 insert_idx = _physmap_idx;
593 for (i = 0; i <= _physmap_idx; i += 2) {
594 if (base < physmap[i + 1]) {
595 if (base + length <= physmap[i]) {
599 if (boothowto & RB_VERBOSE)
601 "Overlapping memory regions, ignoring second region\n");
606 /* See if we can prepend to the next entry. */
607 if (insert_idx <= _physmap_idx &&
608 base + length == physmap[insert_idx]) {
609 physmap[insert_idx] = base;
613 /* See if we can append to the previous entry. */
614 if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
615 physmap[insert_idx - 1] += length;
620 *physmap_idxp = _physmap_idx;
621 if (_physmap_idx == PHYSMAP_SIZE) {
623 "Too many segments in the physical address map, giving up\n");
628 * Move the last 'N' entries down to make room for the new
631 for (i = _physmap_idx; i > insert_idx; i -= 2) {
632 physmap[i] = physmap[i - 2];
633 physmap[i + 1] = physmap[i - 1];
636 /* Insert the new entry. */
637 physmap[insert_idx] = base;
638 physmap[insert_idx + 1] = base + length;
642 #define efi_next_descriptor(ptr, size) \
643 ((struct efi_md *)(((uint8_t *) ptr) + size))
646 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
649 struct efi_md *map, *p;
654 static const char *types[] = {
660 "RuntimeServicesCode",
661 "RuntimeServicesData",
662 "ConventionalMemory",
667 "MemoryMappedIOPortSpace",
672 * Memory map data provided by UEFI via the GetMemoryMap
675 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
676 map = (struct efi_md *)((uint8_t *)efihdr + efisz);
678 if (efihdr->descriptor_size == 0)
680 ndesc = efihdr->memory_size / efihdr->descriptor_size;
682 if (boothowto & RB_VERBOSE)
683 printf("%23s %12s %12s %8s %4s\n",
684 "Type", "Physical", "Virtual", "#Pages", "Attr");
686 for (i = 0, p = map; i < ndesc; i++,
687 p = efi_next_descriptor(p, efihdr->descriptor_size)) {
688 if (boothowto & RB_VERBOSE) {
689 if (p->md_type <= EFI_MD_TYPE_PALCODE)
690 type = types[p->md_type];
693 printf("%23s %012lx %12p %08lx ", type, p->md_phys,
694 p->md_virt, p->md_pages);
695 if (p->md_attr & EFI_MD_ATTR_UC)
697 if (p->md_attr & EFI_MD_ATTR_WC)
699 if (p->md_attr & EFI_MD_ATTR_WT)
701 if (p->md_attr & EFI_MD_ATTR_WB)
703 if (p->md_attr & EFI_MD_ATTR_UCE)
705 if (p->md_attr & EFI_MD_ATTR_WP)
707 if (p->md_attr & EFI_MD_ATTR_RP)
709 if (p->md_attr & EFI_MD_ATTR_XP)
711 if (p->md_attr & EFI_MD_ATTR_RT)
716 switch (p->md_type) {
717 case EFI_MD_TYPE_CODE:
718 case EFI_MD_TYPE_DATA:
719 case EFI_MD_TYPE_BS_CODE:
720 case EFI_MD_TYPE_BS_DATA:
721 case EFI_MD_TYPE_FREE:
723 * We're allowed to use any entry with these types.
730 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE),
731 physmap, physmap_idxp))
738 try_load_dtb(caddr_t kmdp)
742 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
743 if (dtbp == (vm_offset_t)NULL) {
744 printf("ERROR loading DTB\n");
748 if (OF_install(OFW_FDT, 0) == FALSE)
749 panic("Cannot install FDT");
751 if (OF_init((void *)dtbp) != 0)
752 panic("OF_init failed with the found device tree");
759 int dcache_line_shift, icache_line_shift;
762 ctr_el0 = READ_SPECIALREG(ctr_el0);
764 /* Read the log2 words in each D cache line */
765 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
766 /* Get the D cache line size */
767 dcache_line_size = sizeof(int) << dcache_line_shift;
769 /* And the same for the I cache */
770 icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
771 icache_line_size = sizeof(int) << icache_line_shift;
773 idcache_line_size = MIN(dcache_line_size, icache_line_size);
777 initarm(struct arm64_bootparams *abp)
779 struct efi_map_header *efihdr;
781 vm_offset_t lastaddr;
786 /* Set the module data location */
787 preload_metadata = (caddr_t)(uintptr_t)(abp->modulep);
789 /* Find the kernel address */
790 kmdp = preload_search_by_type("elf kernel");
792 kmdp = preload_search_by_type("elf64 kernel");
794 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
795 kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *);
801 /* Find the address to start allocating from */
802 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t);
804 /* Load the physical memory ranges */
806 efihdr = (struct efi_map_header *)preload_search_info(kmdp,
807 MODINFO_METADATA | MODINFOMD_EFI_MAP);
808 add_efi_map_entries(efihdr, physmap, &physmap_idx);
810 /* Print the memory map */
812 for (i = 0; i < physmap_idx; i += 2)
813 mem_len += physmap[i + 1] - physmap[i];
815 /* Set the pcpu data, this is needed by pmap_bootstrap */
817 pcpu_init(pcpup, 0, sizeof(struct pcpu));
820 * Set the pcpu pointer with a backup in tpidr_el1 to be
821 * loaded when entering the kernel from userland.
825 "msr tpidr_el1, %0" :: "r"(pcpup));
827 PCPU_SET(curthread, &thread0);
829 /* Do basic tuning, hz etc */
834 /* Bootstrap enough of pmap to enter the kernel proper */
835 pmap_bootstrap(abp->kern_l1pt, KERNBASE - abp->kern_delta,
836 lastaddr - KERNBASE);
838 arm_devmap_bootstrap(0, NULL);
842 init_proc0(abp->kern_stack);
843 msgbufinit(msgbufp, msgbufsize);
845 init_param2(physmem);