2 * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
18 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
20 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
22 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
23 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "opt_hwpmc_hooks.h"
33 #include <machine/asm.h>
34 #include <machine/hid.h>
35 #include <machine/param.h>
36 #include <machine/spr.h>
37 #include <machine/pte.h>
38 #include <machine/trap.h>
39 #include <machine/vmparam.h>
40 #include <machine/tlb.h>
42 #define TMPSTACKSZ 16384
45 #define GET_TOCBASE(r) \
47 #define TOC_RESTORE nop
57 #define THREAD_REG %r13
62 #define GET_TOCBASE(r)
73 #define THREAD_REG %r2
84 * This symbol is here for the benefit of kvm_mkdb, and is supposed to
85 * mark the start of kernel text.
91 * Startup entry. Note, this must be the first thing in the text segment!
98 * Assumptions on the boot loader:
99 * - System memory starts from physical address 0
100 * - It's mapped by a single TLB1 entry
101 * - TLB1 mapping is 1:1 pa to va
102 * - Kernel is loaded at 64MB boundary
103 * - All PID registers are set to the same value
104 * - CPU is running in AS=0
106 * Registers contents provided by the loader(8):
108 * r3 : metadata pointer
110 * We rearrange the TLB1 layout as follows:
111 * - Find TLB1 entry we started in
112 * - Make sure it's protected, invalidate other entries
113 * - Create temp entry in the second AS (make sure it's not TLB[1])
114 * - Switch to temp mapping
115 * - Map 64MB of RAM in TLB1[1]
116 * - Use AS=0, set EPN to VM_MIN_KERNEL_ADDRESS and RPN to kernel load address
117 * - Switch to TLB1[1] mapping
118 * - Invalidate temp mapping
120 * locore registers use:
122 * r2 : trace pointer (AP only, for early diagnostics)
123 * r3-r27 : scratch registers
124 * r28 : temp TLB1 entry
125 * r29 : initial TLB1 entry we started in
126 * r30-r31 : arguments (metadata pointer)
130 * Keep arguments in r30 & r31 for later use.
138 li %r3, PSL_DE /* Keep debug exceptions for CodeWarrior. */
140 oris %r3, %r3, PSL_CM@h
146 * Initial HIDs configuration
150 rlwinm %r3, %r3, 16, 16, 31
152 lis %r4, HID0_E500_DEFAULT_SET@h
153 ori %r4, %r4, HID0_E500_DEFAULT_SET@l
155 /* Check for e500mc and e5500 */
156 cmpli 0, 0, %r3, FSL_E500mc
159 lis %r4, HID0_E500MC_DEFAULT_SET@h
160 ori %r4, %r4, HID0_E500MC_DEFAULT_SET@l
163 cmpli 0, 0, %r3, FSL_E5500
166 lis %r4, HID0_E5500_DEFAULT_SET@h
167 ori %r4, %r4, HID0_E5500_DEFAULT_SET@l
174 * E500mc and E5500 do not have HID1 register, so skip HID1 setup on
177 cmpli 0, 0, %r3, FSL_E500mc
179 cmpli 0, 0, %r3, FSL_E5500
181 cmpli 0, 0, %r3, FSL_E6500
184 lis %r3, HID1_E500_DEFAULT_SET@h
185 ori %r3, %r3, HID1_E500_DEFAULT_SET@l
189 /* Invalidate all entries in TLB0 */
197 * Locate the TLB1 entry that maps this code
201 bl tlb1_find_current /* the entry found is returned in r29 */
203 bl tlb1_inval_all_but_current
206 * Create temporary mapping in AS=1 and switch to it
208 bl tlb1_temp_mapping_as1
211 ori %r3, %r3, (PSL_IS | PSL_DS)
214 addi %r4, %r4, (3f - 2b)
217 rfi /* Switch context */
220 * Invalidate initial entry
227 * Setup final mapping in TLB1[1] and switch to it
229 /* Final kernel mapping, map in 64 MB of RAM */
230 lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */
231 li %r4, 0 /* Entry 0 */
232 rlwimi %r3, %r4, 16, 10, 15
236 li %r3, (TLB_SIZE_64M << MAS1_TSIZE_SHIFT)@l
237 oris %r3, %r3, (MAS1_VALID | MAS1_IPROT)@h
238 mtspr SPR_MAS1, %r3 /* note TS was not filled, so it's TS=0 */
241 LOAD_ADDR(%r3, VM_MIN_KERNEL_ADDRESS)
242 ori %r3, %r3, (_TLB_ENTRY_SHARED | MAS2_M)@l /* WIMGE = 0b00100 */
246 /* Discover phys load address */
248 3: mflr %r4 /* Use current address */
249 rlwinm %r4, %r4, 0, 0, 5 /* 64MB alignment mask */
250 ori %r4, %r4, (MAS3_SX | MAS3_SW | MAS3_SR)@l
251 mtspr SPR_MAS3, %r4 /* Set RPN and protection */
260 /* Switch to the above TLB1[1] mapping */
267 rlwinm %r4, %r4, 0, 6, 31 /* Current offset from kernel load address */
268 rlwinm %r3, %r3, 0, 0, 19
270 add %r4, %r4, %r3 /* Convert to kernel virtual address */
271 addi %r4, %r4, (5f - 4b)
272 li %r3, PSL_DE /* Note AS=0 */
274 oris %r3, %r3, PSL_CM@h
281 * Invalidate temp mapping
290 /* Set up the TOC pointer */
295 .llong __tocbase + 0x8000 - .
301 /* Get load offset */
302 ld %r31,-0x8000(%r2) /* First TOC entry is TOC base */
303 subf %r31,%r31,%r2 /* Subtract from real TOC base to get base */
305 /* Set up the stack pointer */
306 addis %r1,%r2,TOC_REF(tmpstack)@ha
307 ld %r1,TOC_REF(tmpstack)@l(%r1)
308 addi %r1,%r1,TMPSTACKSZ-96
318 * Setup a temporary stack
325 addi %r1, %r1, (TMPSTACKSZ - 16)
332 .long _GLOBAL_OFFSET_TABLE_-.
334 lwz %r3,0(%r5) /* _DYNAMIC in %r3 */
336 lwz %r4,4(%r5) /* GOT pointer */
338 lwz %r4,4(%r4) /* got[0] is _DYNAMIC link addr */
339 subf %r4,%r4,%r3 /* subtract to calculate relocbase */
341 bl CNAME(elf_reloc_self)
345 * Initialise exception vector offsets
351 * Set up arguments and jump to system initialization code
360 /* Switch to thread0.td_kstack now */
365 /* Machine independet part, does not return */
373 /************************************************************************/
375 /************************************************************************/
391 * Initial configuration
394 mflr %r31 /* r31 hold the address of bp_trace */
398 rlwinm %r3, %r3, 16, 16, 31
400 /* HID0 for E500 is default */
401 lis %r4, HID0_E500_DEFAULT_SET@h
402 ori %r4, %r4, HID0_E500_DEFAULT_SET@l
404 cmpli 0, 0, %r3, FSL_E500mc
406 lis %r4, HID0_E500MC_DEFAULT_SET@h
407 ori %r4, %r4, HID0_E500MC_DEFAULT_SET@l
410 cmpli 0, 0, %r3, FSL_E5500
412 lis %r4, HID0_E5500_DEFAULT_SET@h
413 ori %r4, %r4, HID0_E5500_DEFAULT_SET@l
418 /* Enable branch prediction */
423 /* Invalidate all entries in TLB0 */
428 * Find TLB1 entry which is translating us now
432 bl tlb1_find_current /* the entry number found is in r29 */
434 bl tlb1_inval_all_but_current
437 * Create temporary translation in AS=1 and switch to it
440 bl tlb1_temp_mapping_as1
443 ori %r3, %r3, (PSL_IS | PSL_DS)
445 oris %r3, %r3, PSL_CM@h
449 addi %r4, %r4, (4f - 3b)
452 rfi /* Switch context */
455 * Invalidate initial entry
462 * Setup final mapping in TLB1[1] and switch to it
464 /* Final kernel mapping, map in 64 MB of RAM */
465 lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */
466 li %r4, 0 /* Entry 0 */
467 rlwimi %r3, %r4, 16, 4, 15
471 li %r3, (TLB_SIZE_64M << MAS1_TSIZE_SHIFT)@l
472 oris %r3, %r3, (MAS1_VALID | MAS1_IPROT)@h
473 mtspr SPR_MAS1, %r3 /* note TS was not filled, so it's TS=0 */
476 LOAD_ADDR(%r3, VM_MIN_KERNEL_ADDRESS)
477 ori %r3, %r3, (_TLB_ENTRY_SHARED | MAS2_M)@l /* WIMGE = 0b00100 */
481 /* Retrieve kernel load [physical] address from bp_kernload */
499 rlwinm %r3, %r3, 0, 0, 19
501 sub %r4, %r4, %r5 /* offset of bp_kernload within __boot_page */
504 /* Set RPN and protection */
505 ori %r3, %r3, (MAS3_SX | MAS3_SW | MAS3_SR)@l
515 /* Switch to the final mapping */
518 rlwinm %r3, %r3, 0, 0xfff /* Offset from boot page start */
519 add %r3, %r3, %r5 /* Make this virtual address */
520 addi %r3, %r3, (7f - 6b)
522 lis %r4, PSL_CM@h /* Note AS=0 */
524 li %r4, 0 /* Note AS=0 */
532 * At this point we're running at virtual addresses VM_MIN_KERNEL_ADDRESS and
533 * beyond so it's allowed to directly access all locations the kernel was linked
538 * Invalidate temp mapping
544 /* Set up the TOC pointer */
549 .llong __tocbase + 0x8000 - .
555 /* Set up the stack pointer */
556 addis %r1,%r2,TOC_REF(tmpstack)@ha
557 ld %r1,TOC_REF(tmpstack)@l(%r1)
558 addi %r1,%r1,TMPSTACKSZ-96
561 * Setup a temporary stack
569 addi %r1, %r1, (TMPSTACKSZ - 16)
573 * Initialise exception vector offsets
579 * Assign our pcpu instance
589 bl CNAME(pmap_bootstrap_ap)
592 bl CNAME(cpudep_ap_bootstrap)
594 /* Switch to the idle thread's kstack */
597 bl CNAME(machdep_ap_bootstrap)
604 #if defined (BOOKE_E500)
606 * Invalidate all entries in the given TLB.
611 rlwinm %r3, %r3, 3, (1 << 3) /* TLBSEL */
612 ori %r3, %r3, (1 << 2) /* INVALL */
622 * expects address to look up in r3, returns entry number in r29
624 * FIXME: the hidden assumption is we are now running in AS=0, but we should
625 * retrieve actual AS from MSR[IS|DS] and put it in MAS6[SAS]
629 slwi %r17, %r17, MAS6_SPID0_SHIFT
634 rlwinm %r29, %r17, 16, 26, 31 /* MAS0[ESEL] -> r29 */
636 /* Make sure we have IPROT set on the entry */
638 oris %r17, %r17, MAS1_IPROT@h
647 * Invalidates a single entry in TLB1.
653 lis %r4, MAS0_TLBSEL1@h /* Select TLB1 */
654 rlwimi %r4, %r3, 16, 10, 15 /* Select our entry */
658 li %r5, 0 /* MAS1[V] = 0 */
667 * r29 current entry number
668 * r28 returned temp entry
671 tlb1_temp_mapping_as1:
672 /* Read our current translation */
673 lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */
674 rlwimi %r3, %r29, 16, 10, 15 /* Select our current entry */
680 * Prepare and write temp entry
682 * FIXME this is not robust against overflow i.e. when the current
683 * entry is the last in TLB1
685 lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */
686 addi %r28, %r29, 1 /* Use next entry. */
687 rlwimi %r3, %r28, 16, 10, 15 /* Select temp entry */
692 rlwimi %r5, %r4, 12, 19, 19
693 li %r4, 0 /* Global mapping, TID=0 */
694 rlwimi %r5, %r4, 16, 8, 15
695 oris %r5, %r5, (MAS1_VALID | MAS1_IPROT)@h
709 * Loops over TLB1, invalidates all entries skipping the one which currently
715 tlb1_inval_all_but_current:
716 mfspr %r3, SPR_TLB1CFG /* Get number of entries */
717 andi. %r3, %r3, TLBCFG_NENTRY_MASK@l
718 li %r4, 0 /* Start from Entry 0 */
719 1: lis %r5, MAS0_TLBSEL1@h
720 rlwimi %r5, %r4, 16, 10, 15
725 cmpw %r4, %r29 /* our current entry? */
727 rlwinm %r5, %r5, 0, 2, 31 /* clear VALID and IPROT bits */
734 cmpw %r4, %r3 /* Check if this is the last entry */
739 * MAS7 and MAS8 conditional zeroing.
744 rlwinm %r20, %r20, 16, 16, 31
745 cmpli 0, 0, %r20, FSL_E500v1
756 rlwinm %r20, %r20, 16, 16, 31
757 cmpli 0, 0, %r20, FSL_E500mc
759 cmpli 0, 0, %r20, FSL_E5500
772 * The __boot_tlb1 table is used to hold BSP TLB1 entries
773 * marked with _TLB_ENTRY_SHARED flag during AP bootstrap.
774 * The BSP fills in the table in tlb_ap_prep() function. Next,
775 * AP loads its contents to TLB1 hardware in pmap_bootstrap_ap().
778 .space TLB1_MAX_ENTRIES * TLB_ENTRY_SIZE
782 * Boot page needs to be exactly 4K, with the last word of this page
783 * acting as the reset vector, so we need to stuff the remainder.
784 * Upon release from holdoff CPU fetches the last word of the boot
787 .space 4092 - (__boot_page_padding - __boot_page)
791 /************************************************************************/
792 /* locore subroutines */
793 /************************************************************************/
796 * Cache disable/enable/inval sequences according
797 * to section 2.16 of E500CORE RM.
800 /* Invalidate d-cache */
801 mfspr %r3, SPR_L1CSR0
802 ori %r3, %r3, (L1CSR0_DCFI | L1CSR0_DCLFR)@l
805 mtspr SPR_L1CSR0, %r3
807 1: mfspr %r3, SPR_L1CSR0
808 andi. %r3, %r3, L1CSR0_DCFI
812 ENTRY(dcache_disable)
813 /* Disable d-cache */
814 mfspr %r3, SPR_L1CSR0
820 mtspr SPR_L1CSR0, %r3
826 mfspr %r3, SPR_L1CSR0
827 oris %r3, %r3, (L1CSR0_DCPE | L1CSR0_DCE)@h
828 ori %r3, %r3, (L1CSR0_DCPE | L1CSR0_DCE)@l
831 mtspr SPR_L1CSR0, %r3
836 /* Invalidate i-cache */
837 mfspr %r3, SPR_L1CSR1
838 ori %r3, %r3, (L1CSR1_ICFI | L1CSR1_ICLFR)@l
840 mtspr SPR_L1CSR1, %r3
842 1: mfspr %r3, SPR_L1CSR1
843 andi. %r3, %r3, L1CSR1_ICFI
847 ENTRY(icache_disable)
848 /* Disable i-cache */
849 mfspr %r3, SPR_L1CSR1
854 mtspr SPR_L1CSR1, %r3
860 mfspr %r3, SPR_L1CSR1
861 oris %r3, %r3, (L1CSR1_ICPE | L1CSR1_ICE)@h
862 ori %r3, %r3, (L1CSR1_ICPE | L1CSR1_ICE)@l
864 mtspr SPR_L1CSR1, %r3
869 * L2 cache disable/enable/inval sequences for E500mc.
873 mfspr %r3, SPR_L2CSR0
874 oris %r3, %r3, (L2CSR0_L2FI | L2CSR0_L2LFC)@h
875 ori %r3, %r3, (L2CSR0_L2FI | L2CSR0_L2LFC)@l
877 mtspr SPR_L2CSR0, %r3
879 1: mfspr %r3, SPR_L2CSR0
880 andis. %r3, %r3, L2CSR0_L2FI@h
884 ENTRY(l2cache_enable)
885 mfspr %r3, SPR_L2CSR0
886 oris %r3, %r3, (L2CSR0_L2E | L2CSR0_L2PE)@h
888 mtspr SPR_L2CSR0, %r3
893 * Branch predictor setup.
897 ori %r3, %r3, BUCSR_BBFI
901 ori %r3, %r3, BUCSR_BPEN
908 * XXX: This should be moved to a shared AIM/booke asm file, if one ever is
915 /************************************************************************/
917 /************************************************************************/
920 GLOBAL(__startkernel)
928 .space 10240 /* XXX: this really should not be necessary */
931 TOC_ENTRY(bp_kernload)
935 * Compiled KERNBASE locations
938 .set kernbase, KERNBASE
940 #include <powerpc/booke/trap_subr.S>