2 * Copyright (c) 2001 Jake Burkholder.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <machine/asm.h>
28 __FBSDID("$FreeBSD$");
30 #include <sys/errno.h>
32 #include <machine/asi.h>
33 #include <machine/asmacros.h>
34 #include <machine/hypervisorvar.h>
35 #include <machine/intr_machdep.h>
36 #include <machine/ktr.h>
37 #include <machine/pstate.h>
41 .register %g2, #ignore
42 .register %g3, #ignore
43 .register %g6, #ignore
48 * Generate load and store instructions for the corresponding width and asi
49 * (or not). Note that we want to evaluate the macro args before
50 * concatenating, so that E really turns into nothing.
52 #define _LD(w, a) ld ## w ## a
53 #define _ST(w, a) st ## w ## a
55 #define LD(w, a) _LD(w, a)
56 #define ST(w, a) _ST(w, a)
59 * Common code for copy routines.
61 * We use large macros to generate functions for each of the copy routines.
62 * This allows the load and store instructions to be generated for the right
63 * operation, asi or not. It is possible to write an asi independent function
64 * but this would require 2 expensive wrs in the main loop to switch %asi.
65 * It would also screw up profiling (if we ever get it), but may save some I$.
66 * We assume that either one of dasi and sasi is empty, or that they are both
67 * the same (empty or non-empty). It is up to the caller to set %asi.
71 * ASI independent implementation of copystr(9).
72 * Used to implement copyinstr() and copystr().
74 * Return value is in %g1.
76 #define _COPYSTR(src, dst, len, done, sa, sasi, da, dasi) \
82 LD(ub, sa) [src] sasi, %g1 ; \
83 ST(b, da) %g1, [dst] dasi ; \
88 2: mov ENAMETOOLONG, %g1 ; \
89 3: sub src, %g2, %g2 ; \
95 * ASI independent implementation of memset(3).
96 * Used to implement bzero(), memset() and aszero().
98 * If the pattern is non-zero, duplicate it to fill 64 bits.
99 * Store bytes until dst is 8-byte aligned, then store 8 bytes.
100 * It has yet to be determined how much unrolling is beneficial.
101 * Could also read and compare before writing to minimize snoop traffic.
103 * XXX bzero() should be implemented as
104 * #define bzero(dst, len) (void)memset((dst), 0, (len))
107 #define _MEMSET(dst, pat, len, da, dasi) \
109 and pat, 0xff, pat ; \
113 sllx pat, 16, %g1 ; \
115 sllx pat, 32, %g1 ; \
123 ST(b, da) pat, [dst] dasi ; \
130 ST(x, da) pat, [dst] dasi ; \
131 ST(x, da) pat, [dst + 8] dasi ; \
132 ST(x, da) pat, [dst + 16] dasi ; \
133 ST(x, da) pat, [dst + 24] dasi ; \
140 ST(x, da) pat, [dst] dasi ; \
147 ST(b, da) pat, [dst] dasi ; \
153 * ASI independent implementation of memcpy(3).
154 * Used to implement bcopy(), copyin(), copyout(), memcpy(), ascopy(),
155 * ascopyfrom() and ascopyto().
157 * Transfer bytes until dst is 8-byte aligned. If src is then also 8 byte
158 * aligned, transfer 8 bytes, otherwise finish with bytes. The unaligned
159 * case could be optimized, but it is expected that this is the uncommon
160 * case and of questionable value. The code to do so is also rather large
161 * and ugly. It has yet to be determined how much unrolling is beneficial.
163 * XXX bcopy() must also check for overlap. This is stupid.
164 * XXX bcopy() should be implemented as
165 * #define bcopy(src, dst, len) (void)memcpy((dst), (src), (len))
168 #define _MEMCPY(dst, src, len, da, dasi, sa, sasi) \
174 LD(ub, sa) [src] sasi, %g1 ; \
175 ST(b, da) %g1, [dst] dasi ; \
188 LD(x, sa) [src] sasi, %g1 ; \
189 LD(x, sa) [src + 8] sasi, %g2 ; \
190 LD(x, sa) [src + 16] sasi, %g3 ; \
191 LD(x, sa) [src + 24] sasi, %g4 ; \
192 ST(x, da) %g1, [dst] dasi ; \
193 ST(x, da) %g2, [dst + 8] dasi ; \
194 ST(x, da) %g3, [dst + 16] dasi ; \
195 ST(x, da) %g4, [dst + 24] dasi ; \
203 LD(x, sa) [src] sasi, %g1 ; \
204 ST(x, da) %g1, [dst] dasi ; \
212 LD(ub, sa) [src] sasi, %g1 ; \
213 ST(b, da) %g1, [dst] dasi ; \
220 * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len)
224 _MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi)
230 * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len)
234 _MEMCPY(%o2, %o1, %o3, E, E, a, %asi)
240 * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len)
244 _MEMCPY(%o2, %o0, %o3, a, %asi, E, E)
250 * void aszero(u_long asi, vm_offset_t pa, size_t len)
254 _MEMSET(%o1, %g0, %o2, a, %asi)
260 * int bcmp(const void *b1, const void *b2, size_t len)
265 1: ldub [%o0 + %o3], %o4
266 ldub [%o1 + %o3], %o5
278 * void bcopy(const void *src, void *dst, size_t len)
282 * Check for overlap, and copy backwards if so.
306 * Do the fast version.
308 3: _MEMCPY(%o1, %o0, %o2, E, E, E, E)
314 * void bzero(void *b, size_t len)
317 _MEMSET(%o0, %g0, %o1, E, E)
323 * int copystr(const void *src, void *dst, size_t len, size_t *done)
326 _COPYSTR(%o0, %o1, %o2, %o3, E, E, E, E)
332 * void *memcpy(void *dst, const void *src, size_t len)
336 _MEMCPY(%o3, %o1, %o2, E, E, E, E)
342 * void *memset(void *b, int c, size_t len)
346 _MEMSET(%o3, %o1, %o2, E, E)
351 .globl copy_nofault_begin
356 * int copyin(const void *uaddr, void *kaddr, size_t len)
359 wr %g0, ASI_AIUS, %asi
360 _MEMCPY(%o1, %o0, %o2, E, E, a, %asi)
366 * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
369 wr %g0, ASI_AIUS, %asi
370 _COPYSTR(%o0, %o1, %o2, %o3, a, %asi, E, E)
376 * int copyout(const void *kaddr, void *uaddr, size_t len)
379 wr %g0, ASI_AIUS, %asi
380 _MEMCPY(%o1, %o0, %o2, a, %asi, E, E)
385 .globl copy_nofault_end
394 .globl fs_nofault_begin
399 * Chatty aliases for fetch, store functions.
401 .globl fubyte, fusword, fuword, subyte, susword, suword
403 .set fusword, fuword16
404 .set fuword, fuword64
406 .set susword, suword16
407 .set suword, suword64
409 .globl casuword, fuptr, suptr
410 .set casuword, casuword64
415 * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s)
418 casa [%o0] ASI_AIUS, %o1, %o2
424 * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s)
427 casxa [%o0] ASI_AIUS, %o1, %o2
433 * int fuword8(const void *base)
437 lduba [%o0] ASI_AIUS, %o0
441 * int fuword16(const void *base)
445 lduha [%o0] ASI_AIUS, %o0
449 * int32_t fuword32(const void *base)
453 lduwa [%o0] ASI_AIUS, %o0
457 * int64_t fuword64(const void *base)
461 ldxa [%o0] ASI_AIUS, %o0
465 * int suword8(const void *base, int word)
468 stba %o1, [%o0] ASI_AIUS
474 * int suword16(const void *base, int word)
477 stha %o1, [%o0] ASI_AIUS
483 * int suword32(const void *base, int32_t word)
486 stwa %o1, [%o0] ASI_AIUS
492 * int suword64(const void *base, int64_t word)
495 stxa %o1, [%o0] ASI_AIUS
500 .globl fs_nofault_intr_begin
501 fs_nofault_intr_begin:
505 * int fuswintr(const void *base)
509 lduha [%o0] ASI_AIUS, %o0
513 * int suswintr(const void *base, int word)
516 stha %o1, [%o0] ASI_AIUS
521 .globl fs_nofault_intr_end
525 .globl fs_nofault_end
534 .globl fas_nofault_begin
538 * int fasword8(u_long asi, uint64_t addr, uint8_t *val)
543 lduba [%o1] %asi, %o3
551 * int fasword16(u_long asi, uint64_t addr, uint16_t *val)
556 lduha [%o1] %asi, %o3
564 * int fasword32(u_long asi, uint64_t addr, uint32_t *val)
569 lduwa [%o1] %asi, %o3
576 .globl fas_nofault_end
586 .globl fpu_fault_begin
594 .globl fpu_fault_size
595 .set fpu_fault_size, fpu_fault_end - fpu_fault_begin
601 ldx [%g1 + _JB_FP], %g2
606 ldx [%g1 + _JB_SP], %o2
610 ldx [%g1 + _JB_PC], %o7
613 2: PANIC("longjmp botch", %l1)
617 stx %sp, [%o0 + _JB_SP]
618 stx %o7, [%o0 + _JB_PC]
619 stx %fp, [%o0 + _JB_FP]
625 * void openfirmware(cell_t args[])
628 save %sp, -CCFSZ, %sp
629 SET(ofw_vec, %l7, %l6)
632 wrpr %g0, PIL_TICK, %pil
637 restore %o0, %g0, %o0
641 /* SUN4V_FIXME - uses a now illegal ASI */
643 * void ofw_exit(cell_t args[])
645 ENTRY(openfirmware_exit)
646 save %sp, -CCFSZ, %sp
648 wrpr %g0, PIL_TICK, %pil
649 SET(ofw_tba, %l7, %l5)
651 wrpr %l5, 0, %tba ! restore the ofw trap table
652 SET(ofw_vec, %l7, %l6)
654 SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l7, %l0)
655 sub %l0, SPOFF, %fp ! setup a stack in a locked page
656 sub %l0, SPOFF + CCFSZ, %sp
657 mov AA_DMMU_PCXR, %l3 ! set context 0
658 stxa %g0, [%l3] ASI_DMMU
660 wrpr %g0, 0, %tl ! force trap level 0
664 END(openfirmware_exit)
667 ENTRY(set_mmfsa_scratchpad)
668 stxa %o0, [%g0]ASI_SCRATCHPAD
671 END(set_mmfsa_scratchpad)
672 ENTRY(set_pcpu_scratchpad)
673 mov SCRATCH_REG_PCPU, %g2
674 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD
677 END(set_pcpu_scratchpad)
678 ENTRY(set_hash_kernel_scratchpad)
679 mov SCRATCH_REG_HASH_KERNEL, %g2
680 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD
683 END(set_hash_kernel_scratchpad)
684 ENTRY(set_tsb_kernel_scratchpad)
685 mov SCRATCH_REG_TSB_KERNEL, %g2
686 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD
689 END(set_tsb_kernel_scratchpad)
691 ENTRY(set_hash_user_scratchpad)
692 mov SCRATCH_REG_HASH_USER, %g2
693 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD
696 END(set_hash_user_scratchpad)
697 ENTRY(set_tsb_user_scratchpad)
698 mov SCRATCH_REG_TSB_USER, %g2
699 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD
702 END(set_tsb_user_scratchpad)
706 * %o0 ra - real address
707 * [%o1] lo - lower 64-bits
708 * [%o2] hi - upper 64-bits
712 save %sp, -SA(MINFRAME64), %sp
713 ldda [%g0 + %i0]ASI_LDTD_REAL, %l0
730 mov MAP_ITLB | MAP_DTLB, %o4
731 mov MMU_DEMAP_PAGE, %o5
735 ba,pt %xcc, panic_bad_hcall
736 mov MMU_DEMAP_PAGE, %o1
757 mov MAP_ITLB | MAP_DTLB, %o3
758 mov MMU_DEMAP_CTX, %o5
762 ba,pt %xcc, panic_bad_hcall
763 mov MMU_DEMAP_CTX, %o1
772 mov MAP_ITLB | MAP_DTLB, %o2
773 mov MMU_DEMAP_ALL, %o5
777 ba,pt %xcc, panic_bad_hcall
778 mov MMU_DEMAP_ALL, %o1
786 * panic_bad_hcall is called when a hcall returns
794 .asciz "hypervisor call 0x%x returned an unexpected error %d"
797 ENTRY(panic_bad_hcall)
799 sethi %hi(bad_hcall_error), %o0
800 or %o0, %lo(bad_hcall_error), %o0
812 save %sp, -SA(MINFRAME64), %sp
820 1: stxa %g0, [%i0]ASI_REAL
821 stxa %g0, [%i0 + %l1]ASI_REAL
822 stxa %g0, [%i0 + %l2]ASI_REAL
823 stxa %g0, [%i0 + %l3]ASI_REAL
824 stxa %g0, [%i0 + %l4]ASI_REAL
825 stxa %g0, [%i0 + %l5]ASI_REAL
826 stxa %g0, [%i0 + %l6]ASI_REAL
827 stxa %g0, [%i0 + %l7]ASI_REAL
838 ldx [PCPU(MONDO_DATA)], %g2
852 ENTRY(init_mondo_queue)
853 mov CPU_MONDO_QUEUE_TAIL, %g2
854 ldxa [%g2]ASI_QUEUE, %g2
855 mov CPU_MONDO_QUEUE_HEAD, %g6
856 stxa %g2, [%g6]ASI_QUEUE
857 mov DEV_MONDO_QUEUE_TAIL, %g2
858 ldxa [%g2]ASI_QUEUE, %g2
859 mov DEV_MONDO_QUEUE_HEAD, %g6
860 stxa %g2, [%g6]ASI_QUEUE
863 END(init_mondo_queue)
884 * XXX including sys/gmon.h in genassym.c is not possible due to uintfptr_t
888 #define GMON_PROF_OFF 3
889 #define GMON_PROF_HIRES 4
892 .set _mcount, __cyg_profile_func_enter
894 ENTRY(__cyg_profile_func_enter)
895 SET(_gmonparam, %o3, %o2)
896 lduw [%o2 + GM_STATE], %o3
897 cmp %o3, GMON_PROF_OFF
900 SET(mcount, %o3, %o2)
905 END(__cyg_profile_func_enter)
909 ENTRY(__cyg_profile_func_exit)
910 SET(_gmonparam, %o3, %o2)
911 lduw [%o2 + GM_STATE], %o3
912 cmp %o3, GMON_PROF_HIRES
915 SET(mexitcount, %o3, %o2)
920 END(__cyg_profile_func_exit)