2 * Copyright (c) 2012-2014 Andrew Turner
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include "opt_kstack_pages.h"
31 #include <sys/syscall.h>
32 #include <machine/asm.h>
33 #include <machine/armreg.h>
34 #include <machine/hypervisor.h>
35 #include <machine/param.h>
36 #include <machine/pte.h>
37 #include <machine/vm.h>
38 #include <machine/vmparam.h>
42 #if PAGE_SIZE == PAGE_SIZE_16K
44 * The number of level 3 tables to create. 32 will allow for 1G of address
45 * space, the same as a single level 2 page with 4k pages.
47 #define L3_PAGE_COUNT 32
51 .set kernbase, KERNBASE
55 * MMU on with an identity map, or off
58 * We are loaded at a 2MiB aligned address
66 * Disable the MMU. We may have entered the kernel with it on and
67 * will need to update the tables later. If this has been set up
68 * with anything other than a VA == PA map then this will fail,
69 * but in this case the code to find where we are running from
70 * would have also failed.
78 /* Set the context id */
79 msr contextidr_el1, xzr
81 /* Get the virt -> phys offset */
87 * x28 = Our physical load address
90 /* Create the page tables */
96 * x26 = Kernel L1 table
103 /* Load the new ttbr0 pagetable */
104 adrp x27, pagetable_l0_ttbr0
105 add x27, x27, :lo12:pagetable_l0_ttbr0
107 /* Jump to the virtual address space */
112 /* Set up the stack */
113 adrp x25, initstack_end
114 add x25, x25, :lo12:initstack_end
116 sub sp, sp, #PCB_SIZE
126 #if defined(PERTHREAD_SSP)
127 /* Set sp_el0 to the boot canary for early per-thread SSP to work */
128 adrp x15, boot_canary
129 add x15, x15, :lo12:boot_canary
133 /* Backup the module pointer */
136 sub sp, sp, #BOOTPARAMS_SIZE
139 /* Degate the delda so it is VA -> PA */
142 str x1, [x0, #BP_MODULEP]
143 str x29, [x0, #BP_KERN_DELTA]
145 add x25, x25, :lo12:initstack
146 str x25, [x0, #BP_KERN_STACK]
147 str x27, [x0, #BP_KERN_TTBR0]
148 str x23, [x0, #BP_BOOT_EL]
150 /* trace back starts here */
152 /* Branch to C code */
154 /* We are done with the boot params */
155 add sp, sp, #BOOTPARAMS_SIZE
158 * Enable pointer authentication in the kernel. We set the keys for
159 * thread0 in initarm so have to wait until it returns to enable it.
160 * If we were to enable it in initarm then any authentication when
161 * returning would fail as it was called with pointer authentication
168 /* We should not get here */
182 * mpentry(unsigned long)
184 * Called by a core when it is being brought online.
185 * The data in x0 is passed straight to init_secondary.
188 /* Disable interrupts */
189 msr daifset, #DAIF_INTR
194 /* Set the context id */
195 msr contextidr_el1, xzr
197 /* Load the kernel page table */
198 adrp x24, pagetable_l0_ttbr1
199 add x24, x24, :lo12:pagetable_l0_ttbr1
200 /* Load the identity page table */
201 adrp x27, pagetable_l0_ttbr0_boostrap
202 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap
207 /* Load the new ttbr0 pagetable */
208 adrp x27, pagetable_l0_ttbr0
209 add x27, x27, :lo12:pagetable_l0_ttbr0
211 /* Jump to the virtual address space */
212 ldr x15, =mp_virtdone
216 /* Start using the AP boot stack */
221 #if defined(PERTHREAD_SSP)
222 /* Set sp_el0 to the boot canary for early per-thread SSP to work */
223 adrp x15, boot_canary
224 add x15, x15, :lo12:boot_canary
228 /* Load the kernel ttbr0 pagetable */
232 /* Invalidate the TLB */
242 * If we are started in EL2, configure the required hypervisor
243 * registers and drop to EL1.
253 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it
254 * which may break address translation.
262 /* Configure the Hypervisor */
263 ldr x2, =(HCR_RW | HCR_APK | HCR_API)
266 /* Load the Virtualization Process ID Register */
270 /* Load the Virtualization Multiprocess ID Register */
274 /* Set the bits that need to be 1 in sctlr_el1 */
278 /* Don't trap to EL2 for exceptions */
282 /* Don't trap to EL2 for CP15 traps */
285 /* Enable access to the physical timers at EL1 */
287 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN)
290 /* Set the counter offset to a known value */
293 /* Hypervisor trap functions */
294 adrp x2, hyp_stub_vectors
295 add x2, x2, :lo12:hyp_stub_vectors
298 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */
301 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h)
304 /* Configure GICv3 CPU interface */
305 mrs x2, id_aa64pfr0_el1
306 /* Extract GIC bits from the register */
307 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
308 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */
309 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT)
313 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */
314 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */
318 /* Set the address to return to our return address */
330 * Get the delta between the physical address we were loaded to and the
331 * virtual address we expect to run from. This is used when building the
332 * initial page table.
334 LENTRY(get_virt_delta)
335 /* Load the physical address of virt_map */
337 add x29, x29, :lo12:virt_map
338 /* Load the virtual address of virt_map stored in virt_map */
340 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */
342 /* Find the load address for the kernel */
353 * This builds the page tables containing the identity map, and the kernel
357 * We were loaded to an address that is on a 2MiB boundary
358 * All the memory must not cross a 1GiB boundaty
359 * x28 contains the physical address we were loaded from
361 * TODO: This is out of date.
362 * There are at least 5 pages before that address for the page tables
363 * The pages used are:
364 * - The Kernel L2 table
365 * - The Kernel L1 table
366 * - The Kernel L0 table (TTBR1)
367 * - The identity (PA = VA) L1 table
368 * - The identity (PA = VA) L0 table (TTBR0)
370 LENTRY(create_pagetables)
371 /* Save the Link register */
374 /* Clean the page table */
376 add x6, x6, :lo12:pagetable
378 adrp x27, pagetable_end
379 add x27, x27, :lo12:pagetable_end
381 stp xzr, xzr, [x6], #16
382 stp xzr, xzr, [x6], #16
383 stp xzr, xzr, [x6], #16
384 stp xzr, xzr, [x6], #16
389 * Build the TTBR1 maps.
392 /* Find the size of the kernel */
395 #if defined(LINUX_BOOT_ABI)
396 /* X19 is used as 'map FDT data' flag */
399 /* No modules or FDT pointer ? */
403 * Test if x0 points to modules descriptor(virtual address) or
404 * to FDT (physical address)
406 cmp x0, x6 /* x6 is #(KERNBASE) */
410 /* Booted with modules pointer */
411 /* Find modulep - begin */
414 * Add space for the module data. When PAGE_SIZE is 4k this will
415 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is
416 * larger it will be at least as large as we use smaller level 3
419 ldr x7, =((6 * 1024 * 1024) - 1)
423 #if defined(LINUX_BOOT_ABI)
425 /* Booted by U-Boot booti with FDT data */
426 /* Set 'map FDT data' flag */
430 /* Booted by U-Boot booti without FTD data */
431 /* Find the end - begin */
436 * Add one 2MiB page for copy of FDT data (maximum FDT size),
437 * one for metadata and round up
439 ldr x7, =(3 * L2_SIZE - 1)
444 #if PAGE_SIZE != PAGE_SIZE_4K
446 * Create L3 pages. The kernel will be loaded at a 2M aligned
447 * address, however L2 blocks are too large when the page size is
448 * not 4k to map the kernel with such an aligned address. However,
449 * when the page size is larger than 4k, L2 blocks are too large to
450 * map the kernel with such an alignment.
453 /* Get the number of l3 pages to allocate, rounded down */
454 lsr x10, x8, #(L3_SHIFT)
456 /* Create the kernel space L2 table */
458 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
461 bl build_l3_page_pagetable
463 /* Move to the l2 table */
464 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT)
467 /* Link the l2 -> l3 table */
472 /* Get the number of l2 pages to allocate, rounded down */
473 lsr x10, x8, #(L2_SHIFT)
475 /* Create the kernel space L2 table */
477 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
480 bl build_l2_block_pagetable
483 /* Move to the l1 table */
484 add x26, x26, #PAGE_SIZE
486 /* Link the l1 -> l2 table */
491 /* Move to the l0 table */
492 add x24, x26, #PAGE_SIZE
494 /* Link the l0 -> l1 table */
501 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG.
502 * They are only needed early on, so the VA = PA map is uncached.
504 add x27, x24, #PAGE_SIZE
506 mov x6, x27 /* The initial page table */
508 /* Create the VA = PA map */
509 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
511 and x16, x16, #(~L2_OFFSET)
512 mov x9, x16 /* PA start */
513 mov x8, x16 /* VA start (== PA start) */
515 bl build_l2_block_pagetable
517 #if defined(SOCDEV_PA)
518 /* Create a table for the UART */
519 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE))
521 add x16, x16, x9 /* VA start */
524 /* Store the socdev virtual address */
525 add x17, x8, #(SOCDEV_PA & L2_OFFSET)
527 str x17, [x9, :lo12:socdev_va]
529 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */
531 bl build_l2_block_pagetable
534 #if defined(LINUX_BOOT_ABI)
538 /* Create the mapping for FDT data (2 MiB max) */
539 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
541 add x16, x16, x9 /* VA start */
543 mov x9, x0 /* PA start */
544 /* Update the module pointer to point at the allocated memory */
545 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */
546 add x0, x0, x8 /* Add the aligned virtual address */
549 bl build_l2_block_pagetable
554 /* Move to the l1 table */
555 add x27, x27, #PAGE_SIZE
557 /* Link the l1 -> l2 table */
562 /* Move to the l0 table */
563 add x27, x27, #PAGE_SIZE
565 /* Link the l0 -> l1 table */
571 /* Restore the Link register */
574 LEND(create_pagetables)
577 * Builds an L0 -> L1 table descriptor
580 * x8 = Virtual Address
581 * x9 = L1 PA (trashed)
582 * x10 = Entry count (trashed)
583 * x11, x12 and x13 are trashed
585 LENTRY(link_l0_pagetable)
587 * Link an L0 -> L1 table entry.
589 /* Find the table index */
590 lsr x11, x8, #L0_SHIFT
591 and x11, x11, #L0_ADDR_MASK
593 /* Build the L0 block entry */
595 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0)
597 /* Only use the output address bits */
598 lsr x9, x9, #PAGE_SHIFT
599 1: orr x13, x12, x9, lsl #PAGE_SHIFT
601 /* Store the entry */
602 str x13, [x6, x11, lsl #3]
610 LEND(link_l0_pagetable)
613 * Builds an L1 -> L2 table descriptor
616 * x8 = Virtual Address
617 * x9 = L2 PA (trashed)
618 * x11, x12 and x13 are trashed
620 LENTRY(link_l1_pagetable)
622 * Link an L1 -> L2 table entry.
624 /* Find the table index */
625 lsr x11, x8, #L1_SHIFT
626 and x11, x11, #Ln_ADDR_MASK
628 /* Build the L1 block entry */
631 /* Only use the output address bits */
632 lsr x9, x9, #PAGE_SHIFT
633 orr x13, x12, x9, lsl #PAGE_SHIFT
635 /* Store the entry */
636 str x13, [x6, x11, lsl #3]
639 LEND(link_l1_pagetable)
642 * Builds count 2 MiB page table entry
644 * x7 = Block attributes
646 * x9 = PA start (trashed)
647 * x10 = Entry count (trashed)
648 * x11, x12 and x13 are trashed
650 LENTRY(build_l2_block_pagetable)
652 * Build the L2 table entry.
654 /* Find the table index */
655 lsr x11, x8, #L2_SHIFT
656 and x11, x11, #Ln_ADDR_MASK
658 /* Build the L2 block entry */
659 orr x12, x7, #L2_BLOCK
660 orr x12, x12, #(ATTR_DEFAULT)
661 orr x12, x12, #(ATTR_S1_UXN)
663 /* Only use the output address bits */
664 lsr x9, x9, #L2_SHIFT
666 /* Set the physical address for this virtual address */
667 1: orr x13, x12, x9, lsl #L2_SHIFT
669 /* Store the entry */
670 str x13, [x6, x11, lsl #3]
678 LEND(build_l2_block_pagetable)
680 #if PAGE_SIZE != PAGE_SIZE_4K
682 * Builds an L2 -> L3 table descriptor
685 * x8 = Virtual Address
686 * x9 = L3 PA (trashed)
687 * x11, x12 and x13 are trashed
689 LENTRY(link_l2_pagetable)
691 * Link an L2 -> L3 table entry.
693 /* Find the table index */
694 lsr x11, x8, #L2_SHIFT
695 and x11, x11, #Ln_ADDR_MASK
697 /* Build the L1 block entry */
700 /* Only use the output address bits */
701 lsr x9, x9, #PAGE_SHIFT
702 orr x13, x12, x9, lsl #PAGE_SHIFT
704 /* Store the entry */
705 str x13, [x6, x11, lsl #3]
708 LEND(link_l2_pagetable)
711 * Builds count level 3 page table entries
713 * x7 = Block attributes
715 * x9 = PA start (trashed)
716 * x10 = Entry count (trashed)
717 * x11, x12 and x13 are trashed
719 LENTRY(build_l3_page_pagetable)
721 * Build the L3 table entry.
723 /* Find the table index */
724 lsr x11, x8, #L3_SHIFT
725 and x11, x11, #Ln_ADDR_MASK
727 /* Build the L3 page entry */
728 orr x12, x7, #L3_PAGE
729 orr x12, x12, #(ATTR_DEFAULT)
730 orr x12, x12, #(ATTR_S1_UXN)
732 /* Only use the output address bits */
733 lsr x9, x9, #L3_SHIFT
735 /* Set the physical address for this virtual address */
736 1: orr x13, x12, x9, lsl #L3_SHIFT
738 /* Store the entry */
739 str x13, [x6, x11, lsl #3]
747 LEND(build_l3_page_pagetable)
753 /* Load the exception vectors */
754 ldr x2, =exception_vectors
757 /* Load ttbr0 and ttbr1 */
762 /* Clear the Monitor Debug System control register */
765 /* Invalidate the TLB */
774 * Setup TCR according to the PARange and ASIDBits fields
775 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the
776 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS
777 * to 1 only if the ASIDBits field equals 0b0010.
780 mrs x3, id_aa64mmfr0_el1
782 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
783 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH)
784 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK)
786 /* Check if the HW supports 16 bit ASIDS */
787 cmp x3, #(ID_AA64MMFR0_ASIDBits_16)
788 /* If so x3 == 1, else x3 == 0 */
790 /* Set TCR.AS with x3 */
791 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH)
794 * Check if the HW supports access flag and dirty state updates,
795 * and set TCR_EL1.HA and TCR_EL1.HD accordingly.
797 mrs x3, id_aa64mmfr1_el1
798 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK)
801 orr x2, x2, #(TCR_HA)
806 orr x2, x2, #(TCR_HA | TCR_HD)
816 bic x1, x1, x3 /* Clear the required bits */
817 orr x1, x1, x2 /* Set the required bits */
825 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \
826 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \
827 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \
828 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \
829 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE)
831 #if PAGE_SIZE == PAGE_SIZE_4K
832 #define TCR_TG (TCR_TG1_4K | TCR_TG0_4K)
833 #elif PAGE_SIZE == PAGE_SIZE_16K
834 #define TCR_TG (TCR_TG1_16K | TCR_TG0_16K)
836 #error Unsupported page size
839 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \
840 TCR_CACHE_ATTRS | TCR_SMP_ATTRS)
843 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \
844 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
845 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \
846 SCTLR_M | SCTLR_CP15BEN)
849 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \
857 .section .init_pagetable, "aw", %nobits
860 * 6 initial tables (in the following order):
861 * L2 for kernel (High addresses)
864 * L1 bootstrap for user (Low addresses)
865 * L0 bootstrap for user
868 .globl pagetable_l0_ttbr1
870 #if PAGE_SIZE != PAGE_SIZE_4K
871 .space (PAGE_SIZE * L3_PAGE_COUNT)
879 pagetable_l2_ttbr0_bootstrap:
881 pagetable_l1_ttbr0_bootstrap:
883 pagetable_l0_ttbr0_boostrap:
895 .space (PAGE_SIZE * KSTACK_PAGES)
900 EENTRY(aarch32_sigcode)
901 .word 0xe1a0000d // mov r0, sp
902 .word 0xe2800040 // add r0, r0, #SIGF_UC
903 .word 0xe59f700c // ldr r7, [pc, #12]
904 .word 0xef000000 // swi #0
905 .word 0xe59f7008 // ldr r7, [pc, #8]
906 .word 0xef000000 // swi #0
907 .word 0xeafffffa // b . - 16
908 EEND(aarch32_sigcode)
914 .global sz_aarch32_sigcode
916 .quad aarch32_esigcode - aarch32_sigcode