2 * Copyright (c) 1998 Doug Rabson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
30 * All rights reserved.
32 * Author: Chris G. Demetriou
34 * Permission to use, copy, modify and distribute this software and
35 * its documentation is hereby granted, provided that both the copyright
36 * notice and this permission notice appear in all copies of the
37 * software, derivative works or modified versions, and any portions
38 * thereof, and that both notices appear in supporting documentation.
40 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
41 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
42 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
44 * Carnegie Mellon requests users of this software to return to
46 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
47 * School of Computer Science
48 * Carnegie Mellon University
49 * Pittsburgh PA 15213-3890
51 * any improvements or extensions that they make and grant Carnegie the
52 * rights to redistribute these changes.
55 #include <machine/asm.h>
56 #include <machine/ia64_cpu.h>
62 * ia64_change_mode: change mode to/from physical mode
65 * r14 psr for desired mode
69 * ar.bsp tranlated to new mode
71 ENTRY_NOPROFILE(ia64_change_mode, 0)
73 mov r19=ar.rsc // save rsc while we change mode
74 tbit.nz p6,p7=r14,17 // physical or virtual ?
76 mov ar.rsc=0 // turn off RSE
77 (p6) mov r15=7 // RR base for virtual addresses
78 (p7) mov r15=0 // RR base for physical addresses
80 flushrs // no dirty registers please
87 dep r16=r15,r16,61,3 // new address of ar.bsp
88 dep r17=r15,r17,61,3 // new address of rp
89 dep sp=r15,sp,61,3 // new address of sp
96 mov cr.ipsr=r14 // psr for new mode
98 add r16=2f-1b,r16 // address to rfi to
100 dep r16=r15,r16,61,3 // new mode address for rfi
102 mov cr.iip=r16 // setup for rfi
107 2: mov ar.rsc=r19 // restore ar.rsc
108 br.ret.sptk.few rp // now in new mode
109 END(ia64_change_mode)
112 * ia64_physical_mode: change mode to physical mode
115 * ret0 psr to restore
119 * ar.bsp tranlated to physical mode
122 ENTRY(ia64_physical_mode, 0)
125 movl r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH)
128 andcm r14=r14,r15 // clear various xT bits
130 or r14=r14,r16 // make sure BN=1
131 or ret0=ret0,r16 // make sure BN=1
133 br.cond.sptk.many ia64_change_mode
134 END(ia64_physical_mode)
137 * ia64_call_efi_physical: call an EFI procedure in physical mode
140 * in0 Address of EFI procedure descriptor
141 * in1-in5 Arguments to EFI procedure
144 * ret0-ret3 return values from EFI
147 ENTRY(ia64_call_efi_physical, 6)
151 alloc loc0=ar.pfs,6,4,5,0
157 br.call.sptk.many rp=ia64_physical_mode
159 mov loc2=r8 // psr to restore mode
160 mov loc3=gp // save kernel gp
161 ld8 r14=[in0],8 // function address
168 ld8 gp=[in0] // function gp value
172 br.call.sptk.many rp=b6 // call EFI procedure
173 mov gp=loc3 // restore kernel gp
175 mov r14=loc2 // psr to restore mode
176 br.call.sptk.many rp=ia64_change_mode
182 END(ia64_call_efi_physical)
184 /**************************************************************************/
188 st8.rel [r15]=r0 // Clear onfault.
196 * casuword(u_long *p, u_long old, u_long new)
197 * Perform a compare-exchange in user space.
201 add r15=PC_CURTHREAD,r13
202 movl r14=VM_MAX_ADDRESS
206 ld8 r15=[r15] // r15 = curthread
207 cmp.geu p6,p0=in0,r14
217 ld8 r15=[r15] // r15 = PCB
220 add r15=PCB_ONFAULT,r15
224 st8 [r15]=r14 // Set onfault
226 cmpxchg8.rel ret0=[in0],in2,ar.ccv
231 st8.rel [r15]=r0 // Clear onfault
246 * casuword32(uint32_t *p, uint32_t old, uint32_t new)
247 * Perform a 32-bit compare-exchange in user space.
251 add r15=PC_CURTHREAD,r13
252 movl r14=VM_MAX_ADDRESS
256 ld8 r15=[r15] // r15 = curthread
257 cmp.geu p6,p0=in0,r14
267 ld8 r15=[r15] // r15 = PCB
270 add r15=PCB_ONFAULT,r15
274 st8 [r15]=r14 // Set onfault
276 cmpxchg4.rel ret0=[in0],in2,ar.ccv
281 st8.rel [r15]=r0 // Clear onfault
296 * subyte(void *addr, int byte)
297 * suword16(void *addr, int word)
298 * suword32(void *addr, int word)
299 * suword64|suword(void *addr, long word)
300 * Store in user space
305 add r15=PC_CURTHREAD,r13
306 movl r14=VM_MAX_ADDRESS
310 ld8 r15=[r15] // r15 = curthread
311 cmp.geu p6,p0=in0,r14
321 ld8 r15=[r15] // r15 = PCB
324 add r15=PCB_ONFAULT,r15
328 st8 [r15]=r14 // Set onfault
335 st8.rel [r15]=r0 // Clear onfault
351 add r15=PC_CURTHREAD,r13
352 movl r14=VM_MAX_ADDRESS
356 ld8 r15=[r15] // r15 = curthread
357 cmp.geu p6,p0=in0,r14
367 ld8 r15=[r15] // r15 = PCB
370 add r15=PCB_ONFAULT,r15
374 st8 [r15]=r14 // Set onfault
381 st8.rel [r15]=r0 // Clear onfault
397 add r15=PC_CURTHREAD,r13
398 movl r14=VM_MAX_ADDRESS
402 ld8 r15=[r15] // r15 = curthread
403 cmp.geu p6,p0=in0,r14
413 ld8 r15=[r15] // r15 = PCB
416 add r15=PCB_ONFAULT,r15
420 st8 [r15]=r14 // Set onfault
427 st8.rel [r15]=r0 // Clear onfault
444 add r15=PC_CURTHREAD,r13
445 movl r14=VM_MAX_ADDRESS
449 ld8 r15=[r15] // r15 = curthread
450 cmp.geu p6,p0=in0,r14
460 ld8 r15=[r15] // r15 = PCB
463 add r15=PCB_ONFAULT,r15
467 st8 [r15]=r14 // Set onfault
474 st8.rel [r15]=r0 // Clear onfault
489 * fubyte(void *addr, int byte)
490 * fuword16(void *addr, int word)
491 * fuword32(void *addr, int word)
492 * fuword64|fuword(void *addr, long word)
493 * Fetch from user space
498 add r15=PC_CURTHREAD,r13
499 movl r14=VM_MAX_ADDRESS
503 ld8 r15=[r15] // r15 = curthread
504 cmp.geu p6,p0=in0,r14
514 ld8 r15=[r15] // r15 = PCB
517 add r15=PCB_ONFAULT,r15
521 st8 [r15]=r14 // Set onfault
529 st8.rel [r15]=r0 // Clear onfault
544 add r15=PC_CURTHREAD,r13
545 movl r14=VM_MAX_ADDRESS
549 ld8 r15=[r15] // r15 = curthread
550 cmp.geu p6,p0=in0,r14
560 ld8 r15=[r15] // r15 = PCB
563 add r15=PCB_ONFAULT,r15
567 st8 [r15]=r14 // Set onfault
575 st8.rel [r15]=r0 // Clear onfault
590 add r15=PC_CURTHREAD,r13
591 movl r14=VM_MAX_ADDRESS
595 ld8 r15=[r15] // r15 = curthread
596 cmp.geu p6,p0=in0,r14
606 ld8 r15=[r15] // r15 = PCB
609 add r15=PCB_ONFAULT,r15
613 st8 [r15]=r14 // Set onfault
621 st8.rel [r15]=r0 // Clear onfault
637 add r15=PC_CURTHREAD,r13
638 movl r14=VM_MAX_ADDRESS
642 ld8 r15=[r15] // r15 = curthread
643 cmp.geu p6,p0=in0,r14
653 ld8 r15=[r15] // r15 = PCB
656 add r15=PCB_ONFAULT,r15
660 st8 [r15]=r14 // Set onfault
668 st8.rel [r15]=r0 // Clear onfault
682 * fuswintr(void *addr)
683 * suswintr(void *addr)
704 /**************************************************************************/
707 * Copy a null-terminated string within the kernel's address space.
708 * If lenp is not NULL, store the number of chars copied in *lenp
710 * int copystr(char *from, char *to, size_t len, size_t *lenp);
713 mov r14=in2 // r14 = i = len
715 (p6) br.cond.spnt.few 2f // if (len == 0), bail out
717 1: ld1 r15=[in0],1 // read one byte
719 st1 [in1]=r15,1 // write that byte
720 add in2=-1,in2 // len--
725 (p6) br.cond.spnt.few 2f // if (*from == 0), bail out
726 (p7) br.cond.sptk.few 1b // if (len != 0) copy more
728 2: cmp.eq p6,p0=r0,in3
729 (p6) br.cond.dpnt.few 3f // if (lenp != NULL)
730 sub r14=r14,in2 // *lenp = (i - len)
734 3: cmp.eq p6,p0=r0,r15
735 (p6) br.cond.spnt.few 4f // *from == '\0'; leave quietly
737 mov ret0=ENAMETOOLONG // *from != '\0'; error.
740 4: mov ret0=0 // return 0.
748 alloc loc0=ar.pfs,4,3,4,0
753 movl loc2=VM_MAX_ADDRESS // make sure that src addr
755 cmp.geu p6,p0=in0,loc2 // is in user space.
757 (p6) br.cond.spnt.few copyerr // if it's not, error out.
758 movl r14=copyerr // set up fault handler.
759 add r15=PC_CURTHREAD,r13 // find curthread
763 add r15=TD_PCB,r15 // find pcb
767 add loc2=PCB_ONFAULT,r15
776 br.call.sptk.few rp=copystr // do the copy.
777 st8 [loc2]=r0 // kill the fault handler.
778 mov ar.pfs=loc0 // restore ar.pfs
779 mov rp=loc1 // restore ra.
780 br.ret.sptk.few rp // ret0 left over from copystr
784 * Not the fastest bcopy in the world.
787 mov ret0=r0 // return zero for copy{in,out}
789 cmp.le p6,p0=in2,r0 // bail if len <= 0
790 (p6) br.ret.spnt.few rp
792 sub r14=in1,in0 ;; // check for overlap
793 cmp.ltu p6,p0=r14,in2 // dst-src < len
794 (p6) br.cond.spnt.few 5f
796 extr.u r14=in0,0,3 // src & 7
797 extr.u r15=in1,0,3 ;; // dst & 7
798 cmp.eq p6,p0=r14,r15 // different alignment?
799 (p6) br.cond.spnt.few 2f // branch if same alignment
801 1: ld1 r14=[in0],1 ;; // copy bytewise
803 add in2=-1,in2 ;; // len--
805 (p6) br.cond.dptk.few 1b // loop
806 br.ret.sptk.few rp // done
808 2: cmp.eq p6,p0=r14,r0 // aligned?
809 (p6) br.cond.sptk.few 4f
811 3: ld1 r14=[in0],1 ;; // copy bytewise
813 extr.u r15=in0,0,3 // src & 7
814 add in2=-1,in2 ;; // len--
815 cmp.eq p6,p0=r0,in2 // done?
816 cmp.eq p7,p0=r0,r15 ;; // aligned now?
817 (p6) br.ret.spnt.few rp // return if done
818 (p7) br.cond.spnt.few 4f // go to main copy
819 br.cond.sptk.few 3b // more bytes to copy
821 // At this point, in2 is non-zero
824 cmp.ltu p6,p0=in2,r14 ;; // len < 8?
825 (p6) br.cond.spnt.few 1b // byte copy the end
826 ld8 r15=[in0],8 ;; // copy word
828 add in2=-8,in2 ;; // len -= 8
829 cmp.ne p6,p0=r0,in2 // done?
830 (p6) br.cond.spnt.few 4b // again
832 br.ret.sptk.few rp // return
834 // Don't bother optimising overlap case
841 6: ld1 r14=[in0],-1 ;;
845 (p6) br.cond.spnt.few 6b
854 br.cond.sptk.few bcopy
861 alloc loc0=ar.pfs,3,3,3,0
866 movl loc2=VM_MAX_ADDRESS // make sure that src addr
868 cmp.geu p6,p0=in0,loc2 // is in user space.
870 (p6) br.cond.spnt.few copyerr // if it's not, error out.
871 movl r14=copyerr // set up fault handler.
872 add r15=PC_CURTHREAD,r13 // find curthread
876 add r15=TD_PCB,r15 // find pcb
880 add loc2=PCB_ONFAULT,r15
888 br.call.sptk.few rp=bcopy // do the copy.
889 st8 [loc2]=r0 // kill the fault handler.
890 mov ar.pfs=loc0 // restore ar.pfs
891 mov rp=loc1 // restore ra.
892 br.ret.sptk.few rp // ret0 left over from bcopy
899 alloc loc0=ar.pfs,3,3,3,0
904 movl loc2=VM_MAX_ADDRESS // make sure that dest addr
906 cmp.geu p6,p0=in1,loc2 // is in user space.
908 (p6) br.cond.spnt.few copyerr // if it's not, error out.
909 movl r14=copyerr // set up fault handler.
910 add r15=PC_CURTHREAD,r13 // find curthread
914 add r15=TD_PCB,r15 // find pcb
918 add loc2=PCB_ONFAULT,r15
926 br.call.sptk.few rp=bcopy // do the copy.
927 st8 [loc2]=r0 // kill the fault handler.
928 mov ar.pfs=loc0 // restore ar.pfs
929 mov rp=loc1 // restore ra.
930 br.ret.sptk.few rp // ret0 left over from bcopy
934 add r14=PC_CURTHREAD,r13 ;; // find curthread
936 add r14=TD_PCB,r14 ;; // curthread->td_addr
938 add r14=PCB_ONFAULT,r14 ;; // &curthread->td_pcb->pcb_onfault
939 st8 [r14]=r0 // reset fault handler
941 mov ret0=EFAULT // return EFAULT
947 * Important registers:
948 * r8 structure return address
949 * rp our return address
950 * in0 caller's ar.pfs
956 ENTRY_NOPROFILE(_mcount, 4)
957 alloc loc0 = ar.pfs, 4, 3, 2, 0
963 br.call.sptk rp = __mcount
970 add r14 = 2f - 1b, r14