2 * Copyright (c) 1993 The Regents of the University of California.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 #include <machine/asmacros.h>
35 #include <machine/cputypes.h>
36 #include <machine/intr_machdep.h>
37 #include <machine/pmap.h>
38 #include <machine/specialreg.h>
57 #if defined(I586_CPU) && defined(DEV_NPX)
63 .globl intrcnt, eintrcnt
65 .space INTRCNT_COUNT * 4
68 .globl intrnames, eintrnames
70 .space INTRCNT_COUNT * (MAXCOMLEN + 1)
77 * void bzero(void *buf, u_int len)
108 * do 64 byte chunks first
110 * XXX this is probably over-unrolled at least for DX2's
167 * a jump table seems to be faster than a loop or more range reductions
169 * XXX need a const section for non-text
205 #if defined(I586_CPU) && defined(DEV_NPX)
211 * The FPU register method is twice as fast as the integer register
212 * method unless the target is in the L1 cache and we pre-allocate a
213 * cache line for it (then the integer register method is 4-5 times
214 * faster). However, we never pre-allocate cache lines, since that
215 * would make the integer method 25% or more slower for the common
216 * case when the target isn't in either the L1 cache or the L2 cache.
217 * Thus we normally use the FPU register method unless the overhead
218 * would be too large.
220 cmpl $256,%ecx /* empirical; clts, fninit, smsw cost a lot */
224 * The FPU registers may belong to an application or to fastmove()
225 * or to another invocation of bcopy() or ourself in a higher level
226 * interrupt or trap handler. Preserving the registers is
227 * complicated since we avoid it if possible at all levels. We
228 * want to localize the complications even when that increases them.
229 * Here the extra work involves preserving CR0_TS in TS.
230 * `fpcurthread != NULL' is supposed to be the condition that all the
231 * FPU resources belong to an application, but fpcurthread and CR0_TS
232 * aren't set atomically enough for this condition to work in
233 * interrupt handlers.
235 * Case 1: FPU registers belong to the application: we must preserve
236 * the registers if we use them, so we only use the FPU register
237 * method if the target size is large enough to amortize the extra
238 * overhead for preserving them. CR0_TS must be preserved although
239 * it is very likely to end up as set.
241 * Case 2: FPU registers belong to fastmove(): fastmove() currently
242 * makes the registers look like they belong to an application so
243 * that cpu_switch() and savectx() don't have to know about it, so
244 * this case reduces to case 1.
246 * Case 3: FPU registers belong to the kernel: don't use the FPU
247 * register method. This case is unlikely, and supporting it would
248 * be more complicated and might take too much stack.
250 * Case 4: FPU registers don't belong to anyone: the FPU registers
251 * don't need to be preserved, so we always use the FPU register
252 * method. CR0_TS must be preserved although it is very likely to
253 * always end up as clear.
255 cmpl $0,PCPU(FPCURTHREAD)
259 * XXX don't use the FPU for cases 1 and 2, since preemptive
260 * scheduling of ithreads broke these cases. Note that we can
261 * no longer get here from an interrupt handler, since the
262 * context sitch to the interrupt handler will have saved the
265 jmp intreg_i586_bzero
267 cmpl $256+184,%ecx /* empirical; not quite 2*108 more */
269 sarb $1,kernel_fpu_lock
278 sarb $1,kernel_fpu_lock
282 fninit /* XXX should avoid needing this */
287 * Align to an 8 byte boundary (misalignment in the main loop would
288 * cost a factor of >= 2). Avoid jumps (at little cost if it is
289 * already aligned) by always zeroing 8 bytes and using the part up
290 * to the _next_ alignment position.
293 addl %edx,%ecx /* part of %ecx -= new_%edx - %edx */
299 * Similarly align `len' to a multiple of 8.
306 * This wouldn't be any faster if it were unrolled, since the loop
307 * control instructions are much faster than the fstl and/or done
308 * in parallel with it so their overhead is insignificant.
310 fpureg_i586_bzero_loop:
315 jae fpureg_i586_bzero_loop
317 cmpl $0,PCPU(FPCURTHREAD)
320 /* XXX check that the condition for cases 1-2 stayed false. */
328 movb $0xfe,kernel_fpu_lock
334 movb $0xfe,kernel_fpu_lock
339 * `rep stos' seems to be the best method in practice for small
340 * counts. Fancy methods usually take too long to start up due
341 * to cache and BTB misses.
362 #endif /* I586_CPU && defined(DEV_NPX) */
431 /* fillw(pat, base, cnt) */
452 cmpl %ecx,%eax /* overlapping && src < dst? */
454 cld /* nope, copy forwards */
463 addl %ecx,%edi /* copy backwards. */
482 * generic_bcopy(src, dst, cnt)
483 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
494 cmpl %ecx,%eax /* overlapping && src < dst? */
497 shrl $2,%ecx /* copy by 32-bit words */
498 cld /* nope, copy forwards */
502 andl $3,%ecx /* any bytes left? */
511 addl %ecx,%edi /* copy backwards */
515 andl $3,%ecx /* any fractional bytes? */
519 movl 20(%esp),%ecx /* copy remainder by 32-bit words */
531 #if defined(I586_CPU) && defined(DEV_NPX)
541 cmpl %ecx,%eax /* overlapping && src < dst? */
547 sarb $1,kernel_fpu_lock
549 cmpl $0,PCPU(FPCURTHREAD)
552 /* XXX turn off handling of cases 1-2, as above. */
553 movb $0xfe,kernel_fpu_lock
565 fninit /* XXX should avoid needing this */
570 #define DCACHE_SIZE 8192
571 cmpl $(DCACHE_SIZE-512)/2,%ecx
573 movl $(DCACHE_SIZE-512)/2,%ecx
577 jb 5f /* XXX should prefetch if %ecx >= 32 */
598 large_i586_bcopy_loop:
619 jae large_i586_bcopy_loop
625 cmpl $0,PCPU(FPCURTHREAD)
628 /* XXX check that the condition for cases 1-2 stayed false. */
637 movb $0xfe,kernel_fpu_lock
640 * This is a duplicate of the main part of generic_bcopy. See the comments
641 * there. Jumping into generic_bcopy would cost a whole 0-1 cycles and
642 * would mess up high resolution profiling.
679 #endif /* I586_CPU && defined(DEV_NPX) */
682 * Note: memcpy does not support overlapping copies
691 shrl $2,%ecx /* copy by 32-bit words */
692 cld /* nope, copy forwards */
696 andl $3,%ecx /* any bytes left? */
704 /*****************************************************************************/
705 /* copyout and fubyte family */
706 /*****************************************************************************/
708 * Access user memory from inside the kernel. These routines and possibly
709 * the math- and DOS emulators should be the only places that do this.
711 * We have to access the memory with user's permissions, so use a segment
712 * selector with RPL 3. For writes to user space we have to additionally
713 * check the PTE for write permission, because the 386 does not check
714 * write permissions when we are executing with EPL 0. The 486 does check
715 * this if the WP bit is set in CR0, so we can use a simpler version here.
717 * These routines set curpcb->onfault for the time they execute. When a
718 * protection violation occurs inside the functions, the trap handler
719 * returns to *curpcb->onfault instead of the function.
723 * copyout(from_kernel, to_user, len) - MP SAFE
730 ENTRY(generic_copyout)
731 movl PCPU(CURPCB),%eax
732 movl $copyout_fault,PCB_ONFAULT(%eax)
739 testl %ebx,%ebx /* anything to do? */
743 * Check explicitly for non-user addresses. If 486 write protection
744 * is being used, this check is essential because we are in kernel
745 * mode so the h/w does not provide any protection against writing
750 * First, prevent address wrapping.
756 * XXX STOP USING VM_MAXUSER_ADDRESS.
757 * It is an end address, not a max, so every time it is used correctly it
758 * looks like there is an off by one error, and of course it caused an off
759 * by one error in several places.
761 cmpl $VM_MAXUSER_ADDRESS,%eax
764 /* bcopy(%esi, %edi, %ebx) */
767 #if defined(I586_CPU) && defined(DEV_NPX)
785 movl PCPU(CURPCB),%edx
786 movl %eax,PCB_ONFAULT(%edx)
795 movl PCPU(CURPCB),%edx
796 movl $0,PCB_ONFAULT(%edx)
800 #if defined(I586_CPU) && defined(DEV_NPX)
803 * Duplicated from generic_copyout. Could be done a bit better.
805 movl PCPU(CURPCB),%eax
806 movl $copyout_fault,PCB_ONFAULT(%eax)
813 testl %ebx,%ebx /* anything to do? */
817 * Check explicitly for non-user addresses. If 486 write protection
818 * is being used, this check is essential because we are in kernel
819 * mode so the h/w does not provide any protection against writing
824 * First, prevent address wrapping.
830 * XXX STOP USING VM_MAXUSER_ADDRESS.
831 * It is an end address, not a max, so every time it is used correctly it
832 * looks like there is an off by one error, and of course it caused an off
833 * by one error in several places.
835 cmpl $VM_MAXUSER_ADDRESS,%eax
838 /* bcopy(%esi, %edi, %ebx) */
842 * End of duplicated code.
853 #endif /* I586_CPU && defined(DEV_NPX) */
856 * copyin(from_user, to_kernel, len) - MP SAFE
863 ENTRY(generic_copyin)
864 movl PCPU(CURPCB),%eax
865 movl $copyin_fault,PCB_ONFAULT(%eax)
868 movl 12(%esp),%esi /* caddr_t from */
869 movl 16(%esp),%edi /* caddr_t to */
870 movl 20(%esp),%ecx /* size_t len */
873 * make sure address is valid
878 cmpl $VM_MAXUSER_ADDRESS,%edx
881 #if defined(I586_CPU) && defined(DEV_NPX)
886 shrl $2,%ecx /* copy longword-wise */
891 andb $3,%cl /* copy remaining bytes */
895 #if defined(I586_CPU) && defined(DEV_NPX)
902 movl PCPU(CURPCB),%edx
903 movl %eax,PCB_ONFAULT(%edx)
911 movl PCPU(CURPCB),%edx
912 movl $0,PCB_ONFAULT(%edx)
916 #if defined(I586_CPU) && defined(DEV_NPX)
919 * Duplicated from generic_copyin. Could be done a bit better.
921 movl PCPU(CURPCB),%eax
922 movl $copyin_fault,PCB_ONFAULT(%eax)
925 movl 12(%esp),%esi /* caddr_t from */
926 movl 16(%esp),%edi /* caddr_t to */
927 movl 20(%esp),%ecx /* size_t len */
930 * make sure address is valid
935 cmpl $VM_MAXUSER_ADDRESS,%edx
938 * End of duplicated code.
944 pushl %ebx /* XXX prepare for fastmove_fault */
950 #endif /* I586_CPU && defined(DEV_NPX) */
952 #if defined(I586_CPU) && defined(DEV_NPX)
953 /* fastmove(src, dst, len)
956 len in %ecx XXX changed to on stack for profiling
957 uses %eax and %edx for tmp. storage
959 /* XXX use ENTRY() to get profiling. fastmove() is actually a non-entry. */
963 subl $PCB_SAVEFPU_SIZE+3*4,%esp
969 testl $7,%esi /* check if src addr is multiple of 8 */
972 testl $7,%edi /* check if dst addr is multiple of 8 */
975 /* XXX grab FPU context atomically. */
978 /* if (fpcurthread != NULL) { */
979 cmpl $0,PCPU(FPCURTHREAD)
981 /* fnsave(&curpcb->pcb_savefpu); */
982 movl PCPU(CURPCB),%eax
983 fnsave PCB_SAVEFPU(%eax)
984 /* FPCURTHREAD = NULL; */
985 movl $0,PCPU(FPCURTHREAD)
988 /* now we own the FPU. */
991 * The process' FP state is saved in the pcb, but if we get
992 * switched, the cpu_switch() will store our FP state in the
993 * pcb. It should be possible to avoid all the copying for
994 * this, e.g., by setting a flag to tell cpu_switch() to
995 * save the state somewhere else.
997 /* tmp = curpcb->pcb_savefpu; */
1002 movl PCPU(CURPCB),%esi
1003 addl $PCB_SAVEFPU,%esi
1005 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1011 /* stop_emulating(); */
1013 /* fpcurthread = curthread; */
1014 movl PCPU(CURTHREAD),%eax
1015 movl %eax,PCPU(FPCURTHREAD)
1016 movl PCPU(CURPCB),%eax
1018 /* XXX end of atomic FPU context grab. */
1021 movl $fastmove_fault,PCB_ONFAULT(%eax)
1078 /* XXX ungrab FPU context atomically. */
1081 /* curpcb->pcb_savefpu = tmp; */
1085 movl PCPU(CURPCB),%edi
1086 addl $PCB_SAVEFPU,%edi
1089 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1096 /* start_emulating(); */
1100 /* fpcurthread = NULL; */
1101 movl $0,PCPU(FPCURTHREAD)
1103 /* XXX end of atomic FPU context ungrab. */
1108 movl PCPU(CURPCB),%eax
1109 movl $fastmove_tail_fault,PCB_ONFAULT(%eax)
1112 shrl $2,%ecx /* copy longword-wise */
1117 andb $3,%cl /* copy remaining bytes */
1127 /* XXX ungrab FPU context atomically. */
1130 movl PCPU(CURPCB),%edi
1131 addl $PCB_SAVEFPU,%edi
1134 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1141 movl $0,PCPU(FPCURTHREAD)
1143 /* XXX end of atomic FPU context ungrab. */
1146 fastmove_tail_fault:
1153 movl PCPU(CURPCB),%edx
1154 movl $0,PCB_ONFAULT(%edx)
1158 #endif /* I586_CPU && defined(DEV_NPX) */
1161 * casuword. Compare and set user word. Returns -1 or the current value.
1164 ALTENTRY(casuword32)
1166 movl PCPU(CURPCB),%ecx
1167 movl $fusufault,PCB_ONFAULT(%ecx)
1168 movl 4(%esp),%edx /* dst */
1169 movl 8(%esp),%eax /* old */
1170 movl 12(%esp),%ecx /* new */
1172 cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address is valid */
1178 cmpxchgl %ecx,(%edx) /* Compare and set. */
1181 * The old value is in %eax. If the store succeeded it will be the
1182 * value we expected (old) from before the store, otherwise it will
1183 * be the current value.
1186 movl PCPU(CURPCB),%ecx
1187 movl $fusufault,PCB_ONFAULT(%ecx)
1188 movl $0,PCB_ONFAULT(%ecx)
1194 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
1195 * memory. All these functions are MPSAFE.
1200 movl PCPU(CURPCB),%ecx
1201 movl $fusufault,PCB_ONFAULT(%ecx)
1202 movl 4(%esp),%edx /* from */
1204 cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address is valid */
1208 movl $0,PCB_ONFAULT(%ecx)
1214 * fuswintr() and suswintr() are specialized variants of fuword16() and
1215 * suword16(), respectively. They are called from the profiling code,
1216 * potentially at interrupt time. If they fail, that's okay; good things
1217 * will happen later. They always fail for now, until the trap code is
1218 * able to deal with this.
1228 movl PCPU(CURPCB),%ecx
1229 movl $fusufault,PCB_ONFAULT(%ecx)
1232 cmpl $VM_MAXUSER_ADDRESS-2,%edx
1236 movl $0,PCB_ONFAULT(%ecx)
1241 movl PCPU(CURPCB),%ecx
1242 movl $fusufault,PCB_ONFAULT(%ecx)
1245 cmpl $VM_MAXUSER_ADDRESS-1,%edx
1249 movl $0,PCB_ONFAULT(%ecx)
1255 movl PCPU(CURPCB),%ecx
1257 movl %eax,PCB_ONFAULT(%ecx)
1262 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
1263 * All these functions are MPSAFE.
1268 movl PCPU(CURPCB),%ecx
1269 movl $fusufault,PCB_ONFAULT(%ecx)
1272 cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address validity */
1278 movl PCPU(CURPCB),%ecx
1279 movl %eax,PCB_ONFAULT(%ecx)
1285 movl PCPU(CURPCB),%ecx
1286 movl $fusufault,PCB_ONFAULT(%ecx)
1289 cmpl $VM_MAXUSER_ADDRESS-2,%edx /* verify address validity */
1295 movl PCPU(CURPCB),%ecx /* restore trashed register */
1296 movl %eax,PCB_ONFAULT(%ecx)
1301 movl PCPU(CURPCB),%ecx
1302 movl $fusufault,PCB_ONFAULT(%ecx)
1305 cmpl $VM_MAXUSER_ADDRESS-1,%edx /* verify address validity */
1311 movl PCPU(CURPCB),%ecx /* restore trashed register */
1312 movl %eax,PCB_ONFAULT(%ecx)
1317 * copyinstr(from, to, maxlen, int *lencopied) - MP SAFE
1319 * copy a string from from to to, stop when a 0 character is reached.
1320 * return ENAMETOOLONG if string is longer than maxlen, and
1321 * EFAULT on protection violations. If lencopied is non-zero,
1322 * return the actual length in *lencopied.
1327 movl PCPU(CURPCB),%ecx
1328 movl $cpystrflt,PCB_ONFAULT(%ecx)
1330 movl 12(%esp),%esi /* %esi = from */
1331 movl 16(%esp),%edi /* %edi = to */
1332 movl 20(%esp),%edx /* %edx = maxlen */
1334 movl $VM_MAXUSER_ADDRESS,%eax
1336 /* make sure 'from' is within bounds */
1340 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
1358 /* Success -- 0 byte reached */
1363 /* edx is zero - return ENAMETOOLONG or EFAULT */
1364 cmpl $VM_MAXUSER_ADDRESS,%esi
1367 movl $ENAMETOOLONG,%eax
1374 /* set *lencopied and return %eax */
1375 movl PCPU(CURPCB),%ecx
1376 movl $0,PCB_ONFAULT(%ecx)
1390 * copystr(from, to, maxlen, int *lencopied) - MP SAFE
1396 movl 12(%esp),%esi /* %esi = from */
1397 movl 16(%esp),%edi /* %edi = to */
1398 movl 20(%esp),%edx /* %edx = maxlen */
1409 /* Success -- 0 byte reached */
1414 /* edx is zero -- return ENAMETOOLONG */
1415 movl $ENAMETOOLONG,%eax
1418 /* set *lencopied and return %eax */
1440 cld /* compare forwards */
1458 * Handling of special 386 registers and descriptor tables etc
1460 /* void lgdt(struct region_descriptor *rdp); */
1463 /* reload the descriptor table */
1468 /* flush the prefetch q */
1472 /* reload "stale" selectors */
1481 /* reload code selector by turning return into intersegmental return */
1489 /* ssdtosd(*ssdp,*sdp) */
1511 /* void reset_dbregs() */
1514 movl %eax,%dr7 /* disable all breapoints first */
1523 /*****************************************************************************/
1524 /* setjump, longjump */
1525 /*****************************************************************************/
1529 movl %ebx,(%eax) /* save ebx */
1530 movl %esp,4(%eax) /* save esp */
1531 movl %ebp,8(%eax) /* save ebp */
1532 movl %esi,12(%eax) /* save esi */
1533 movl %edi,16(%eax) /* save edi */
1534 movl (%esp),%edx /* get rta */
1535 movl %edx,20(%eax) /* save eip */
1536 xorl %eax,%eax /* return(0); */
1542 movl (%eax),%ebx /* restore ebx */
1543 movl 4(%eax),%esp /* restore esp */
1544 movl 8(%eax),%ebp /* restore ebp */
1545 movl 12(%eax),%esi /* restore esi */
1546 movl 16(%eax),%edi /* restore edi */
1547 movl 20(%eax),%edx /* get rta */
1548 movl %edx,(%esp) /* put in return frame */
1549 xorl %eax,%eax /* return(1); */
1555 * Support for BB-profiling (gcc -a). The kernbb program will extract
1556 * the data from the kernel.
1566 NON_GPROF_ENTRY(__bb_init_func)
1575 * Support for reading MSRs in the safe manner.
1578 /* int rdmsr_safe(u_int msr, uint64_t *data) */
1579 movl PCPU(CURPCB),%ecx
1580 movl $msr_onfault,PCB_ONFAULT(%ecx)
1589 movl PCPU(CURPCB),%ecx
1590 movl %eax,PCB_ONFAULT(%ecx)
1595 * Support for writing MSRs in the safe manner.
1598 /* int wrmsr_safe(u_int msr, uint64_t data) */
1599 movl PCPU(CURPCB),%ecx
1600 movl $msr_onfault,PCB_ONFAULT(%ecx)
1608 movl PCPU(CURPCB),%ecx
1609 movl %eax,PCB_ONFAULT(%ecx)
1614 * MSR operations fault handler
1618 movl PCPU(CURPCB),%ecx
1619 movl $0,PCB_ONFAULT(%ecx)