2 * Copyright (c) 2005 Olivier Houchard. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD$");
27 #include <machine/asm.h>
28 #include <sys/param.h>
29 #include <sys/elf32.h>
30 #include <sys/inflate.h>
31 #include <machine/elf.h>
32 #include <machine/pte.h>
33 #include <machine/cpufunc.h>
34 #include <machine/armreg.h>
37 * Since we are compiled outside of the normal kernel build process, we
38 * need to include opt_global.h manually.
40 #include "opt_global.h"
41 #include "opt_kernname.h"
43 extern char kernel_start[];
44 extern char kernel_end[];
55 #define cpu_idcache_wbinv_all arm7tdmi_cache_flushID
56 #elif defined(CPU_ARM8)
57 #define cpu_idcache_wbinv_all arm8_cache_purgeID
58 #elif defined(CPU_ARM9)
59 #define cpu_idcache_wbinv_all arm9_idcache_wbinv_all
60 #elif defined(CPU_ARM9E)
61 #define cpu_idcache_wbinv_all armv5_ec_idcache_wbinv_all
62 #elif defined(CPU_ARM10)
63 #define cpu_idcache_wbinv_all arm10_idcache_wbinv_all
64 #elif defined(CPU_SA110) || defined(CPU_SA1110) || defined(CPU_SA1100) || \
66 #define cpu_idcache_wbinv_all sa1_cache_purgeID
67 #elif defined(CPU_XSCALE_80200) || defined(CPU_XSCALE_80321) || \
68 defined(CPU_XSCALE_PXA2X0) || defined(CPU_XSCALE_IXP425) || \
69 defined(CPU_XSCALE_80219)
70 #define cpu_idcache_wbinv_all xscale_cache_purgeID
71 #elif defined(CPU_XSCALE_81342)
72 #define cpu_idcache_wbinv_all xscalec3_cache_purgeID
74 #ifdef CPU_XSCALE_81342
75 #define cpu_l2cache_wbinv_all xscalec3_l2cache_purge
76 #elif defined(SOC_MV_KIRKWOOD) || defined(SOC_MV_DISCOVERY)
77 #define cpu_l2cache_wbinv_all sheeva_l2cache_wbinv_all
79 #define cpu_l2cache_wbinv_all()
84 int arm_picache_line_size;
87 int arm_pdcache_size; /* and unified */
88 int arm_pdcache_line_size = 32;
92 int arm_pcache_unified;
95 int arm_dcache_align_mask;
97 /* Additional cache information local to this file. Log2 of some of the
99 static int arm_dcache_l2_nsets;
100 static int arm_dcache_l2_assoc;
101 static int arm_dcache_l2_linesize;
104 int block_userspace_access = 0;
105 extern int arm9_dcache_sets_inc;
106 extern int arm9_dcache_sets_max;
107 extern int arm9_dcache_index_max;
108 extern int arm9_dcache_index_inc;
110 static __inline void *
111 memcpy(void *dst, const void *src, int len)
117 if (0 && len >= 4 && !((vm_offset_t)d & 3) &&
118 !((vm_offset_t)s & 3)) {
119 *(uint32_t *)d = *(uint32_t *)s;
132 bzero(void *addr, int count)
134 char *tmp = (char *)addr;
137 if (count >= 4 && !((vm_offset_t)tmp & 3)) {
138 *(uint32_t *)tmp = 0;
149 static void arm9_setup(void);
154 int physaddr = KERNPHYSADDR;
156 unsigned int sp = ((unsigned int)&_end & ~3) + 4;
157 #if defined(FLASHADDR) && defined(LOADERRAMADDR)
160 __asm __volatile("adr %0, _start\n"
162 if ((FLASHADDR > LOADERRAMADDR && pc >= FLASHADDR) ||
163 (FLASHADDR < LOADERRAMADDR && pc < LOADERRAMADDR)) {
165 * We're running from flash, so just copy the whole thing
166 * from flash to memory.
167 * This is far from optimal, we could do the relocation or
168 * the unzipping directly from flash to memory to avoid this
169 * needless copy, but it would require to know the flash
172 unsigned int target_addr;
175 target_addr = (unsigned int)&_start - PHYSADDR + LOADERRAMADDR;
176 tmp_sp = target_addr + 0x100000 +
177 (unsigned int)&_end - (unsigned int)&_start;
178 memcpy((char *)target_addr, (char *)pc,
179 (unsigned int)&_end - (unsigned int)&_start);
180 /* Temporary set the sp and jump to the new location. */
184 : : "r" (target_addr), "r" (tmp_sp));
189 sp += KERNSIZE + 0x100;
190 sp &= ~(L1_TABLE_SIZE - 1);
191 sp += 2 * L1_TABLE_SIZE;
193 sp += 1024 * 1024; /* Should be enough for a stack */
195 __asm __volatile("adr %0, 2f\n"
196 "bic %0, %0, #0xff000000\n"
197 "and %1, %1, #0xff000000\n"
199 "mrc p15, 0, %1, c1, c0, 0\n"
200 "bic %1, %1, #1\n" /* Disable MMU */
201 "orr %1, %1, #(4 | 8)\n" /* Add DC enable,
203 "orr %1, %1, #0x1000\n" /* Add IC enable */
204 "orr %1, %1, #(0x800)\n" /* BPRD enable */
206 "mcr p15, 0, %1, c1, c0, 0\n"
213 : "=r" (tmp1), "+r" (physaddr), "+r" (sp));
216 /* So that idcache_wbinv works; */
217 if ((cpufunc_id() & 0x0000f000) == 0x00009000)
220 cpu_idcache_wbinv_all();
221 cpu_l2cache_wbinv_all();
229 u_int ctype, isize, dsize;
232 __asm __volatile("mrc p15, 0, %0, c0, c0, 1"
236 * ...and thus spake the ARM ARM:
238 * If an <opcode2> value corresponding to an unimplemented or
239 * reserved ID register is encountered, the System Control
240 * processor returns the value of the main ID register.
242 if (ctype == cpufunc_id())
245 if ((ctype & CPU_CT_S) == 0)
246 arm_pcache_unified = 1;
249 * If you want to know how this code works, go read the ARM ARM.
252 arm_pcache_type = CPU_CT_CTYPE(ctype);
253 if (arm_pcache_unified == 0) {
254 isize = CPU_CT_ISIZE(ctype);
255 multiplier = (isize & CPU_CT_xSIZE_M) ? 3 : 2;
256 arm_picache_line_size = 1U << (CPU_CT_xSIZE_LEN(isize) + 3);
257 if (CPU_CT_xSIZE_ASSOC(isize) == 0) {
258 if (isize & CPU_CT_xSIZE_M)
259 arm_picache_line_size = 0; /* not present */
261 arm_picache_ways = 1;
263 arm_picache_ways = multiplier <<
264 (CPU_CT_xSIZE_ASSOC(isize) - 1);
266 arm_picache_size = multiplier << (CPU_CT_xSIZE_SIZE(isize) + 8);
269 dsize = CPU_CT_DSIZE(ctype);
270 multiplier = (dsize & CPU_CT_xSIZE_M) ? 3 : 2;
271 arm_pdcache_line_size = 1U << (CPU_CT_xSIZE_LEN(dsize) + 3);
272 if (CPU_CT_xSIZE_ASSOC(dsize) == 0) {
273 if (dsize & CPU_CT_xSIZE_M)
274 arm_pdcache_line_size = 0; /* not present */
276 arm_pdcache_ways = 1;
278 arm_pdcache_ways = multiplier <<
279 (CPU_CT_xSIZE_ASSOC(dsize) - 1);
281 arm_pdcache_size = multiplier << (CPU_CT_xSIZE_SIZE(dsize) + 8);
283 arm_dcache_align = arm_pdcache_line_size;
285 arm_dcache_l2_assoc = CPU_CT_xSIZE_ASSOC(dsize) + multiplier - 2;
286 arm_dcache_l2_linesize = CPU_CT_xSIZE_LEN(dsize) + 3;
287 arm_dcache_l2_nsets = 6 + CPU_CT_xSIZE_SIZE(dsize) -
288 CPU_CT_xSIZE_ASSOC(dsize) - CPU_CT_xSIZE_LEN(dsize);
290 arm_dcache_align_mask = arm_dcache_align - 1;
297 get_cachetype_cp15();
298 arm9_dcache_sets_inc = 1U << arm_dcache_l2_linesize;
299 arm9_dcache_sets_max = (1U << (arm_dcache_l2_linesize +
300 arm_dcache_l2_nsets)) - arm9_dcache_sets_inc;
301 arm9_dcache_index_inc = 1U << (32 - arm_dcache_l2_assoc);
302 arm9_dcache_index_max = 0U - arm9_dcache_index_inc;
307 static unsigned char *orig_input, *i_input, *i_output;
310 static u_int memcnt; /* Memory allocated: blocks */
311 static size_t memtot; /* Memory allocated: bytes */
313 * Library functions required by inflate().
316 #define MEMSIZ 0x8000
319 * Allocate memory block.
325 static u_char mem[MEMSIZ];
327 if (memtot + size > MEMSIZ)
336 * Free allocated memory block.
354 if ((size_t)(i_input - orig_input) >= KERNCOMPSIZE) {
361 output(void *dummy, unsigned char *ptr, unsigned long len)
365 memcpy(i_output, ptr, len);
371 inflate_kernel(void *kernel, void *startaddr)
374 char slide[GZ_WSIZE];
378 i_input = (char *)kernel + GZ_HEAD;
379 if (((char *)kernel)[3] & 0x18) {
384 i_output = startaddr;
385 bzero(&infl, sizeof(infl));
386 infl.gz_input = input;
387 infl.gz_output = output;
388 infl.gz_slide = slide;
390 return ((char *)(((vm_offset_t)i_output & ~3) + 4));
396 load_kernel(unsigned int kstart, unsigned int curaddr,unsigned int func_end,
400 Elf32_Phdr phdr[64] /* XXX */, *php;
401 Elf32_Shdr shdr[64] /* XXX */;
404 int symtabindex = -1;
405 int symstrindex = -1;
406 vm_offset_t lastaddr = 0;
410 eh = (Elf32_Ehdr *)kstart;
412 entry_point = (void*)eh->e_entry;
413 memcpy(phdr, (void *)(kstart + eh->e_phoff ),
414 eh->e_phnum * sizeof(phdr[0]));
416 /* Determine lastaddr. */
417 for (i = 0; i < eh->e_phnum; i++) {
418 if (lastaddr < (phdr[i].p_vaddr - KERNVIRTADDR + curaddr
420 lastaddr = phdr[i].p_vaddr - KERNVIRTADDR +
421 curaddr + phdr[i].p_memsz;
424 /* Save the symbol tables, as there're about to be scratched. */
425 memcpy(shdr, (void *)(kstart + eh->e_shoff),
426 sizeof(*shdr) * eh->e_shnum);
427 if (eh->e_shnum * eh->e_shentsize != 0 &&
429 for (i = 0; i < eh->e_shnum; i++) {
430 if (shdr[i].sh_type == SHT_SYMTAB) {
431 for (j = 0; j < eh->e_phnum; j++) {
432 if (phdr[j].p_type == PT_LOAD &&
439 shdr[i].sh_offset = 0;
444 if (shdr[i].sh_offset != 0 &&
445 shdr[i].sh_size != 0) {
447 symstrindex = shdr[i].sh_link;
451 func_end = roundup(func_end, sizeof(long));
452 if (symtabindex >= 0 && symstrindex >= 0) {
455 memcpy((void *)func_end, (void *)(
456 shdr[symtabindex].sh_offset + kstart),
457 shdr[symtabindex].sh_size);
458 memcpy((void *)(func_end +
459 shdr[symtabindex].sh_size),
460 (void *)(shdr[symstrindex].sh_offset +
461 kstart), shdr[symstrindex].sh_size);
463 lastaddr += shdr[symtabindex].sh_size;
464 lastaddr = roundup(lastaddr,
465 sizeof(shdr[symtabindex].sh_size));
466 lastaddr += sizeof(shdr[symstrindex].sh_size);
467 lastaddr += shdr[symstrindex].sh_size;
468 lastaddr = roundup(lastaddr,
469 sizeof(shdr[symstrindex].sh_size));
475 return ((void *)lastaddr);
478 for (i = 0; i < j; i++) {
481 if (phdr[i].p_type != PT_LOAD)
483 memcpy((void *)(phdr[i].p_vaddr - KERNVIRTADDR + curaddr),
484 (void*)(kstart + phdr[i].p_offset), phdr[i].p_filesz);
485 /* Clean space from oversized segments, eg: bss. */
486 if (phdr[i].p_filesz < phdr[i].p_memsz)
487 bzero((void *)(phdr[i].p_vaddr - KERNVIRTADDR +
488 curaddr + phdr[i].p_filesz), phdr[i].p_memsz -
491 /* Now grab the symbol tables. */
492 if (symtabindex >= 0 && symstrindex >= 0) {
493 *(Elf_Size *)lastaddr =
494 shdr[symtabindex].sh_size;
495 lastaddr += sizeof(shdr[symtabindex].sh_size);
496 memcpy((void*)lastaddr,
498 shdr[symtabindex].sh_size);
499 lastaddr += shdr[symtabindex].sh_size;
500 lastaddr = roundup(lastaddr,
501 sizeof(shdr[symtabindex].sh_size));
502 *(Elf_Size *)lastaddr =
503 shdr[symstrindex].sh_size;
504 lastaddr += sizeof(shdr[symstrindex].sh_size);
505 memcpy((void*)lastaddr,
507 shdr[symtabindex].sh_size),
508 shdr[symstrindex].sh_size);
509 lastaddr += shdr[symstrindex].sh_size;
510 lastaddr = roundup(lastaddr,
511 sizeof(shdr[symstrindex].sh_size));
512 *(Elf_Addr *)curaddr = MAGIC_TRAMP_NUMBER;
513 *((Elf_Addr *)curaddr + 1) = ssym - curaddr + KERNVIRTADDR;
514 *((Elf_Addr *)curaddr + 2) = lastaddr - curaddr + KERNVIRTADDR;
516 *(Elf_Addr *)curaddr = 0;
517 /* Invalidate the instruction cache. */
518 __asm __volatile("mcr p15, 0, %0, c7, c5, 0\n"
519 "mcr p15, 0, %0, c7, c10, 4\n"
521 __asm __volatile("mrc p15, 0, %0, c1, c0, 0\n"
522 "bic %0, %0, #1\n" /* MMU_ENABLE */
523 "mcr p15, 0, %0, c1, c0, 0\n"
525 /* Jump to the entry point. */
526 ((void(*)(void))(entry_point - KERNVIRTADDR + curaddr))();
527 __asm __volatile(".globl func_end\n"
532 extern char func_end[];
535 #define PMAP_DOMAIN_KERNEL 0 /*
536 * Just define it instead of including the
537 * whole VM headers set.
541 setup_pagetables(unsigned int pt_addr, vm_paddr_t physstart, vm_paddr_t physend,
544 unsigned int *pd = (unsigned int *)pt_addr;
546 int domain = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | DOMAIN_CLIENT;
549 bzero(pd, L1_TABLE_SIZE);
550 for (addr = physstart; addr < physend; addr += L1_S_SIZE) {
551 pd[addr >> L1_S_SHIFT] = L1_TYPE_S|L1_S_C|L1_S_AP(AP_KRW)|
552 L1_S_DOM(PMAP_DOMAIN_KERNEL) | addr;
554 pd[addr >> L1_S_SHIFT] |= L1_S_B;
557 if (0xfff00000 < physstart || 0xfff00000 > physend)
558 pd[0xfff00000 >> L1_S_SHIFT] = L1_TYPE_S|L1_S_AP(AP_KRW)|
559 L1_S_DOM(PMAP_DOMAIN_KERNEL)|physstart;
560 __asm __volatile("mcr p15, 0, %1, c2, c0, 0\n" /* set TTB */
561 "mcr p15, 0, %1, c8, c7, 0\n" /* Flush TTB */
562 "mcr p15, 0, %2, c3, c0, 0\n" /* Set DAR */
563 "mrc p15, 0, %0, c1, c0, 0\n"
564 "orr %0, %0, #1\n" /* MMU_ENABLE */
565 "mcr p15, 0, %0, c1, c0, 0\n"
566 "mrc p15, 0, %0, c2, c0, 0\n" /* CPWAIT */
569 "=r" (tmp) : "r" (pd), "r" (domain));
572 * XXX: This is the most stupid workaround I've ever wrote.
573 * For some reason, the KB9202 won't boot the kernel unless
574 * we access an address which is not in the
575 * 0x20000000 - 0x20ffffff range. I hope I'll understand
576 * what's going on later.
578 __hack = *(volatile int *)0xfffff21c;
586 char *kernel = (char *)&kernel_start;
590 __asm __volatile("mov %0, pc" :
592 curaddr = (void*)((unsigned int)curaddr & 0xfff00000);
594 if (*kernel == 0x1f && kernel[1] == 0x8b) {
595 pt_addr = (((int)&_end + KERNSIZE + 0x100) &
596 ~(L1_TABLE_SIZE - 1)) + L1_TABLE_SIZE;
599 /* So that idcache_wbinv works; */
600 if ((cpufunc_id() & 0x0000f000) == 0x00009000)
603 setup_pagetables(pt_addr, (vm_paddr_t)curaddr,
604 (vm_paddr_t)curaddr + 0x10000000, 1);
606 dst = inflate_kernel(kernel, &_end);
607 kernel = (char *)&_end;
608 altdst = 4 + load_kernel((unsigned int)kernel,
609 (unsigned int)curaddr,
610 (unsigned int)&func_end + 800 , 0);
615 dst = 4 + load_kernel((unsigned int)&kernel_start,
616 (unsigned int)curaddr,
617 (unsigned int)&func_end, 0);
618 dst = (void *)(((vm_offset_t)dst & ~3));
619 pt_addr = ((unsigned int)dst &~(L1_TABLE_SIZE - 1)) + L1_TABLE_SIZE;
620 setup_pagetables(pt_addr, (vm_paddr_t)curaddr,
621 (vm_paddr_t)curaddr + 0x10000000, 0);
622 sp = pt_addr + L1_TABLE_SIZE + 8192;
624 dst = (void *)(sp + 4);
625 memcpy((void *)dst, (void *)&load_kernel, (unsigned int)&func_end -
626 (unsigned int)&load_kernel + 800);
627 do_call(dst, kernel, dst + (unsigned int)(&func_end) -
628 (unsigned int)(&load_kernel) + 800, sp);