2 * Copyright (c) 2012-2014 Andrew Turner
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include "opt_kstack_pages.h"
31 #include <sys/syscall.h>
32 #include <machine/asm.h>
33 #include <machine/armreg.h>
34 #include <machine/hypervisor.h>
35 #include <machine/param.h>
36 #include <machine/pte.h>
37 #include <machine/vm.h>
38 #include <machine/vmparam.h>
42 #if PAGE_SIZE == PAGE_SIZE_16K
44 * The number of level 3 tables to create. 32 will allow for 1G of address
45 * space, the same as a single level 2 page with 4k pages.
47 #define L3_PAGE_COUNT 32
51 .set kernbase, KERNBASE
55 * MMU on with an identity map, or off
58 * We are loaded at a 2MiB aligned address
66 * Disable the MMU. We may have entered the kernel with it on and
67 * will need to update the tables later. If this has been set up
68 * with anything other than a VA == PA map then this will fail,
69 * but in this case the code to find where we are running from
70 * would have also failed.
78 /* Set the context id */
79 msr contextidr_el1, xzr
81 /* Get the virt -> phys offset */
87 * x28 = Our physical load address
90 /* Create the page tables */
96 * x26 = Kernel L1 table
103 /* Load the new ttbr0 pagetable */
104 adrp x27, pagetable_l0_ttbr0
105 add x27, x27, :lo12:pagetable_l0_ttbr0
107 /* Jump to the virtual address space */
112 /* Set up the stack */
113 adrp x25, initstack_end
114 add x25, x25, :lo12:initstack_end
116 sub sp, sp, #PCB_SIZE
126 #if defined(PERTHREAD_SSP)
127 /* Set sp_el0 to the boot canary for early per-thread SSP to work */
128 adrp x15, boot_canary
129 add x15, x15, :lo12:boot_canary
133 /* Backup the module pointer */
136 sub sp, sp, #BOOTPARAMS_SIZE
139 /* Degate the delda so it is VA -> PA */
142 str x1, [x0, #BP_MODULEP]
143 str x29, [x0, #BP_KERN_DELTA]
145 add x25, x25, :lo12:initstack
146 str x25, [x0, #BP_KERN_STACK]
147 str x27, [x0, #BP_KERN_TTBR0]
148 str x23, [x0, #BP_BOOT_EL]
150 /* trace back starts here */
152 /* Branch to C code */
154 /* We are done with the boot params */
155 add sp, sp, #BOOTPARAMS_SIZE
158 * Enable pointer authentication in the kernel. We set the keys for
159 * thread0 in initarm so have to wait until it returns to enable it.
160 * If we were to enable it in initarm then any authentication when
161 * returning would fail as it was called with pointer authentication
168 /* We should not get here */
182 * mpentry(unsigned long)
184 * Called by a core when it is being brought online.
185 * The data in x0 is passed straight to init_secondary.
188 /* Disable interrupts */
189 msr daifset, #DAIF_INTR
194 /* Set the context id */
195 msr contextidr_el1, xzr
197 /* Load the kernel page table */
198 adrp x24, pagetable_l0_ttbr1
199 add x24, x24, :lo12:pagetable_l0_ttbr1
200 /* Load the identity page table */
201 adrp x27, pagetable_l0_ttbr0_boostrap
202 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap
207 /* Load the new ttbr0 pagetable */
208 adrp x27, pagetable_l0_ttbr0
209 add x27, x27, :lo12:pagetable_l0_ttbr0
211 /* Jump to the virtual address space */
212 ldr x15, =mp_virtdone
216 /* Start using the AP boot stack */
221 #if defined(PERTHREAD_SSP)
222 /* Set sp_el0 to the boot canary for early per-thread SSP to work */
223 adrp x15, boot_canary
224 add x15, x15, :lo12:boot_canary
228 /* Load the kernel ttbr0 pagetable */
232 /* Invalidate the TLB */
242 * If we are started in EL2, configure the required hypervisor
243 * registers and drop to EL1.
253 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it
254 * which may break address translation.
262 /* Configure the Hypervisor */
263 ldr x2, =(HCR_RW | HCR_APK | HCR_API)
266 /* Stash value of HCR_EL2 for later */
270 /* Load the Virtualization Process ID Register */
274 /* Load the Virtualization Multiprocess ID Register */
278 /* Set the bits that need to be 1 in sctlr_el1 */
283 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we
284 * don't trap to EL2 for SIMD register usage to have at least a
285 * minimally usable system.
288 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */
289 mov x4, #CPTR_FPEN /* HCR_E2H == 1 */
293 /* Don't trap to EL2 for CP15 traps */
296 /* Enable access to the physical timers at EL1 */
298 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN)
301 /* Set the counter offset to a known value */
304 /* Hypervisor trap functions */
305 adrp x2, hyp_stub_vectors
306 add x2, x2, :lo12:hyp_stub_vectors
309 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */
312 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h)
315 /* Configure GICv3 CPU interface */
316 mrs x2, id_aa64pfr0_el1
317 /* Extract GIC bits from the register */
318 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
319 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */
320 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT)
324 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */
325 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */
329 /* Set the address to return to our return address */
341 * Get the delta between the physical address we were loaded to and the
342 * virtual address we expect to run from. This is used when building the
343 * initial page table.
345 LENTRY(get_virt_delta)
346 /* Load the physical address of virt_map */
348 add x29, x29, :lo12:virt_map
349 /* Load the virtual address of virt_map stored in virt_map */
351 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */
353 /* Find the load address for the kernel */
364 * This builds the page tables containing the identity map, and the kernel
368 * We were loaded to an address that is on a 2MiB boundary
369 * All the memory must not cross a 1GiB boundaty
370 * x28 contains the physical address we were loaded from
372 * TODO: This is out of date.
373 * There are at least 5 pages before that address for the page tables
374 * The pages used are:
375 * - The Kernel L2 table
376 * - The Kernel L1 table
377 * - The Kernel L0 table (TTBR1)
378 * - The identity (PA = VA) L1 table
379 * - The identity (PA = VA) L0 table (TTBR0)
381 LENTRY(create_pagetables)
382 /* Save the Link register */
385 /* Clean the page table */
387 add x6, x6, :lo12:pagetable
389 adrp x27, pagetable_end
390 add x27, x27, :lo12:pagetable_end
392 stp xzr, xzr, [x6], #16
393 stp xzr, xzr, [x6], #16
394 stp xzr, xzr, [x6], #16
395 stp xzr, xzr, [x6], #16
400 * Build the TTBR1 maps.
403 /* Find the size of the kernel */
406 #if defined(LINUX_BOOT_ABI)
407 /* X19 is used as 'map FDT data' flag */
410 /* No modules or FDT pointer ? */
414 * Test if x0 points to modules descriptor(virtual address) or
415 * to FDT (physical address)
417 cmp x0, x6 /* x6 is #(KERNBASE) */
421 /* Booted with modules pointer */
422 /* Find modulep - begin */
425 * Add space for the module data. When PAGE_SIZE is 4k this will
426 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is
427 * larger it will be at least as large as we use smaller level 3
430 ldr x7, =((6 * 1024 * 1024) - 1)
434 #if defined(LINUX_BOOT_ABI)
436 /* Booted by U-Boot booti with FDT data */
437 /* Set 'map FDT data' flag */
441 /* Booted by U-Boot booti without FTD data */
442 /* Find the end - begin */
447 * Add one 2MiB page for copy of FDT data (maximum FDT size),
448 * one for metadata and round up
450 ldr x7, =(3 * L2_SIZE - 1)
455 #if PAGE_SIZE != PAGE_SIZE_4K
457 * Create L3 pages. The kernel will be loaded at a 2M aligned
458 * address, however L2 blocks are too large when the page size is
459 * not 4k to map the kernel with such an aligned address. However,
460 * when the page size is larger than 4k, L2 blocks are too large to
461 * map the kernel with such an alignment.
464 /* Get the number of l3 pages to allocate, rounded down */
465 lsr x10, x8, #(L3_SHIFT)
467 /* Create the kernel space L2 table */
469 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
472 bl build_l3_page_pagetable
474 /* Move to the l2 table */
475 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT)
478 /* Link the l2 -> l3 table */
483 /* Get the number of l2 pages to allocate, rounded down */
484 lsr x10, x8, #(L2_SHIFT)
486 /* Create the kernel space L2 table */
488 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
491 bl build_l2_block_pagetable
494 /* Move to the l1 table */
495 add x26, x26, #PAGE_SIZE
497 /* Link the l1 -> l2 table */
502 /* Move to the l0 table */
503 add x24, x26, #PAGE_SIZE
505 /* Link the l0 -> l1 table */
512 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG.
513 * They are only needed early on, so the VA = PA map is uncached.
515 add x27, x24, #PAGE_SIZE
517 mov x6, x27 /* The initial page table */
519 /* Create the VA = PA map */
520 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
522 and x16, x16, #(~L2_OFFSET)
523 mov x9, x16 /* PA start */
524 mov x8, x16 /* VA start (== PA start) */
526 bl build_l2_block_pagetable
528 #if defined(SOCDEV_PA)
529 /* Create a table for the UART */
530 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE))
532 add x16, x16, x9 /* VA start */
535 /* Store the socdev virtual address */
536 add x17, x8, #(SOCDEV_PA & L2_OFFSET)
538 str x17, [x9, :lo12:socdev_va]
540 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */
542 bl build_l2_block_pagetable
545 #if defined(LINUX_BOOT_ABI)
549 /* Create the mapping for FDT data (2 MiB max) */
550 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
552 add x16, x16, x9 /* VA start */
554 mov x9, x0 /* PA start */
555 /* Update the module pointer to point at the allocated memory */
556 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */
557 add x0, x0, x8 /* Add the aligned virtual address */
560 bl build_l2_block_pagetable
565 /* Move to the l1 table */
566 add x27, x27, #PAGE_SIZE
568 /* Link the l1 -> l2 table */
573 /* Move to the l0 table */
574 add x27, x27, #PAGE_SIZE
576 /* Link the l0 -> l1 table */
582 /* Restore the Link register */
585 LEND(create_pagetables)
588 * Builds an L0 -> L1 table descriptor
591 * x8 = Virtual Address
592 * x9 = L1 PA (trashed)
593 * x10 = Entry count (trashed)
594 * x11, x12 and x13 are trashed
596 LENTRY(link_l0_pagetable)
598 * Link an L0 -> L1 table entry.
600 /* Find the table index */
601 lsr x11, x8, #L0_SHIFT
602 and x11, x11, #L0_ADDR_MASK
604 /* Build the L0 block entry */
606 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0)
608 /* Only use the output address bits */
609 lsr x9, x9, #PAGE_SHIFT
610 1: orr x13, x12, x9, lsl #PAGE_SHIFT
612 /* Store the entry */
613 str x13, [x6, x11, lsl #3]
621 LEND(link_l0_pagetable)
624 * Builds an L1 -> L2 table descriptor
627 * x8 = Virtual Address
628 * x9 = L2 PA (trashed)
629 * x11, x12 and x13 are trashed
631 LENTRY(link_l1_pagetable)
633 * Link an L1 -> L2 table entry.
635 /* Find the table index */
636 lsr x11, x8, #L1_SHIFT
637 and x11, x11, #Ln_ADDR_MASK
639 /* Build the L1 block entry */
642 /* Only use the output address bits */
643 lsr x9, x9, #PAGE_SHIFT
644 orr x13, x12, x9, lsl #PAGE_SHIFT
646 /* Store the entry */
647 str x13, [x6, x11, lsl #3]
650 LEND(link_l1_pagetable)
653 * Builds count 2 MiB page table entry
655 * x7 = Block attributes
657 * x9 = PA start (trashed)
658 * x10 = Entry count (trashed)
659 * x11, x12 and x13 are trashed
661 LENTRY(build_l2_block_pagetable)
663 * Build the L2 table entry.
665 /* Find the table index */
666 lsr x11, x8, #L2_SHIFT
667 and x11, x11, #Ln_ADDR_MASK
669 /* Build the L2 block entry */
670 orr x12, x7, #L2_BLOCK
671 orr x12, x12, #(ATTR_DEFAULT)
672 orr x12, x12, #(ATTR_S1_UXN)
674 /* Only use the output address bits */
675 lsr x9, x9, #L2_SHIFT
677 /* Set the physical address for this virtual address */
678 1: orr x13, x12, x9, lsl #L2_SHIFT
680 /* Store the entry */
681 str x13, [x6, x11, lsl #3]
689 LEND(build_l2_block_pagetable)
691 #if PAGE_SIZE != PAGE_SIZE_4K
693 * Builds an L2 -> L3 table descriptor
696 * x8 = Virtual Address
697 * x9 = L3 PA (trashed)
698 * x11, x12 and x13 are trashed
700 LENTRY(link_l2_pagetable)
702 * Link an L2 -> L3 table entry.
704 /* Find the table index */
705 lsr x11, x8, #L2_SHIFT
706 and x11, x11, #Ln_ADDR_MASK
708 /* Build the L1 block entry */
711 /* Only use the output address bits */
712 lsr x9, x9, #PAGE_SHIFT
713 orr x13, x12, x9, lsl #PAGE_SHIFT
715 /* Store the entry */
716 str x13, [x6, x11, lsl #3]
719 LEND(link_l2_pagetable)
722 * Builds count level 3 page table entries
724 * x7 = Block attributes
726 * x9 = PA start (trashed)
727 * x10 = Entry count (trashed)
728 * x11, x12 and x13 are trashed
730 LENTRY(build_l3_page_pagetable)
732 * Build the L3 table entry.
734 /* Find the table index */
735 lsr x11, x8, #L3_SHIFT
736 and x11, x11, #Ln_ADDR_MASK
738 /* Build the L3 page entry */
739 orr x12, x7, #L3_PAGE
740 orr x12, x12, #(ATTR_DEFAULT)
741 orr x12, x12, #(ATTR_S1_UXN)
743 /* Only use the output address bits */
744 lsr x9, x9, #L3_SHIFT
746 /* Set the physical address for this virtual address */
747 1: orr x13, x12, x9, lsl #L3_SHIFT
749 /* Store the entry */
750 str x13, [x6, x11, lsl #3]
758 LEND(build_l3_page_pagetable)
764 /* Load the exception vectors */
765 ldr x2, =exception_vectors
768 /* Load ttbr0 and ttbr1 */
773 /* Clear the Monitor Debug System control register */
776 /* Invalidate the TLB */
785 * Setup TCR according to the PARange and ASIDBits fields
786 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the
787 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS
788 * to 1 only if the ASIDBits field equals 0b0010.
791 mrs x3, id_aa64mmfr0_el1
793 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
794 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH)
795 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK)
797 /* Check if the HW supports 16 bit ASIDS */
798 cmp x3, #(ID_AA64MMFR0_ASIDBits_16)
799 /* If so x3 == 1, else x3 == 0 */
801 /* Set TCR.AS with x3 */
802 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH)
805 * Check if the HW supports access flag and dirty state updates,
806 * and set TCR_EL1.HA and TCR_EL1.HD accordingly.
808 mrs x3, id_aa64mmfr1_el1
809 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK)
812 orr x2, x2, #(TCR_HA)
817 orr x2, x2, #(TCR_HA | TCR_HD)
827 bic x1, x1, x3 /* Clear the required bits */
828 orr x1, x1, x2 /* Set the required bits */
836 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \
837 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \
838 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \
839 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \
840 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE)
842 #if PAGE_SIZE == PAGE_SIZE_4K
843 #define TCR_TG (TCR_TG1_4K | TCR_TG0_4K)
844 #elif PAGE_SIZE == PAGE_SIZE_16K
845 #define TCR_TG (TCR_TG1_16K | TCR_TG0_16K)
847 #error Unsupported page size
850 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \
851 TCR_CACHE_ATTRS | TCR_SMP_ATTRS)
854 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \
855 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
856 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \
857 SCTLR_M | SCTLR_CP15BEN)
860 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \
868 .section .init_pagetable, "aw", %nobits
871 * 6 initial tables (in the following order):
872 * L2 for kernel (High addresses)
875 * L1 bootstrap for user (Low addresses)
876 * L0 bootstrap for user
879 .globl pagetable_l0_ttbr1
881 #if PAGE_SIZE != PAGE_SIZE_4K
882 .space (PAGE_SIZE * L3_PAGE_COUNT)
890 pagetable_l2_ttbr0_bootstrap:
892 pagetable_l1_ttbr0_bootstrap:
894 pagetable_l0_ttbr0_boostrap:
906 .space (PAGE_SIZE * KSTACK_PAGES)
911 EENTRY(aarch32_sigcode)
912 .word 0xe1a0000d // mov r0, sp
913 .word 0xe2800040 // add r0, r0, #SIGF_UC
914 .word 0xe59f700c // ldr r7, [pc, #12]
915 .word 0xef000000 // swi #0
916 .word 0xe59f7008 // ldr r7, [pc, #8]
917 .word 0xef000000 // swi #0
918 .word 0xeafffffa // b . - 16
919 EEND(aarch32_sigcode)
925 .global sz_aarch32_sigcode
927 .quad aarch32_esigcode - aarch32_sigcode