2 * Copyright (c) 1993 The Regents of the University of California.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 #include <machine/asmacros.h>
39 #include <machine/cputypes.h>
40 #include <machine/pmap.h>
41 #include <machine/specialreg.h>
63 #if defined(I586_CPU) && defined(DEV_NPX)
73 * void bzero(void *buf, u_int len)
98 * do 64 byte chunks first
100 * XXX this is probably over-unrolled at least for DX2's
157 * a jump table seems to be faster than a loop or more range reductions
159 * XXX need a const section for non-text
194 #if defined(I586_CPU) && defined(DEV_NPX)
200 * The FPU register method is twice as fast as the integer register
201 * method unless the target is in the L1 cache and we pre-allocate a
202 * cache line for it (then the integer register method is 4-5 times
203 * faster). However, we never pre-allocate cache lines, since that
204 * would make the integer method 25% or more slower for the common
205 * case when the target isn't in either the L1 cache or the L2 cache.
206 * Thus we normally use the FPU register method unless the overhead
207 * would be too large.
209 cmpl $256,%ecx /* empirical; clts, fninit, smsw cost a lot */
213 * The FPU registers may belong to an application or to fastmove()
214 * or to another invocation of bcopy() or ourself in a higher level
215 * interrupt or trap handler. Preserving the registers is
216 * complicated since we avoid it if possible at all levels. We
217 * want to localize the complications even when that increases them.
218 * Here the extra work involves preserving CR0_TS in TS.
219 * `npxproc != NULL' is supposed to be the condition that all the
220 * FPU resources belong to an application, but npxproc and CR0_TS
221 * aren't set atomically enough for this condition to work in
222 * interrupt handlers.
224 * Case 1: FPU registers belong to the application: we must preserve
225 * the registers if we use them, so we only use the FPU register
226 * method if the target size is large enough to amortize the extra
227 * overhead for preserving them. CR0_TS must be preserved although
228 * it is very likely to end up as set.
230 * Case 2: FPU registers belong to fastmove(): fastmove() currently
231 * makes the registers look like they belong to an application so
232 * that cpu_switch() and savectx() don't have to know about it, so
233 * this case reduces to case 1.
235 * Case 3: FPU registers belong to the kernel: don't use the FPU
236 * register method. This case is unlikely, and supporting it would
237 * be more complicated and might take too much stack.
239 * Case 4: FPU registers don't belong to anyone: the FPU registers
240 * don't need to be preserved, so we always use the FPU register
241 * method. CR0_TS must be preserved although it is very likely to
242 * always end up as clear.
244 cmpl $0,PCPU(NPXPROC)
248 * XXX don't use the FPU for cases 1 and 2, since preemptive
249 * scheduling of ithreads broke these cases. Note that we can
250 * no longer get here from an interrupt handler, since the
251 * context sitch to the interrupt handler will have saved the
254 jmp intreg_i586_bzero
256 cmpl $256+184,%ecx /* empirical; not quite 2*108 more */
258 sarb $1,kernel_fpu_lock
267 sarb $1,kernel_fpu_lock
271 fninit /* XXX should avoid needing this */
276 * Align to an 8 byte boundary (misalignment in the main loop would
277 * cost a factor of >= 2). Avoid jumps (at little cost if it is
278 * already aligned) by always zeroing 8 bytes and using the part up
279 * to the _next_ alignment position.
282 addl %edx,%ecx /* part of %ecx -= new_%edx - %edx */
288 * Similarly align `len' to a multiple of 8.
295 * This wouldn't be any faster if it were unrolled, since the loop
296 * control instructions are much faster than the fstl and/or done
297 * in parallel with it so their overhead is insignificant.
299 fpureg_i586_bzero_loop:
304 jae fpureg_i586_bzero_loop
306 cmpl $0,PCPU(NPXPROC)
309 /* XXX check that the condition for cases 1-2 stayed false. */
317 movb $0xfe,kernel_fpu_lock
323 movb $0xfe,kernel_fpu_lock
328 * `rep stos' seems to be the best method in practice for small
329 * counts. Fancy methods usually take too long to start up due
330 * to cache and BTB misses.
350 #endif /* I586_CPU && defined(DEV_NPX) */
402 /* fillw(pat, base, cnt) */
422 cmpl %ecx,%eax /* overlapping && src < dst? */
424 cld /* nope, copy forwards */
433 addl %ecx,%edi /* copy backwards. */
454 * generic_bcopy(src, dst, cnt)
455 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
466 cmpl %ecx,%eax /* overlapping && src < dst? */
469 shrl $2,%ecx /* copy by 32-bit words */
470 cld /* nope, copy forwards */
474 andl $3,%ecx /* any bytes left? */
483 addl %ecx,%edi /* copy backwards */
487 andl $3,%ecx /* any fractional bytes? */
491 movl 20(%esp),%ecx /* copy remainder by 32-bit words */
502 #if defined(I586_CPU) && defined(DEV_NPX)
512 cmpl %ecx,%eax /* overlapping && src < dst? */
518 sarb $1,kernel_fpu_lock
520 cmpl $0,PCPU(NPXPROC)
523 /* XXX turn off handling of cases 1-2, as above. */
524 movb $0xfe,kernel_fpu_lock
536 fninit /* XXX should avoid needing this */
541 #define DCACHE_SIZE 8192
542 cmpl $(DCACHE_SIZE-512)/2,%ecx
544 movl $(DCACHE_SIZE-512)/2,%ecx
548 jb 5f /* XXX should prefetch if %ecx >= 32 */
569 large_i586_bcopy_loop:
590 jae large_i586_bcopy_loop
596 cmpl $0,PCPU(NPXPROC)
599 /* XXX check that the condition for cases 1-2 stayed false. */
608 movb $0xfe,kernel_fpu_lock
611 * This is a duplicate of the main part of generic_bcopy. See the comments
612 * there. Jumping into generic_bcopy would cost a whole 0-1 cycles and
613 * would mess up high resolution profiling.
649 #endif /* I586_CPU && defined(DEV_NPX) */
652 * Note: memcpy does not support overlapping copies
661 shrl $2,%ecx /* copy by 32-bit words */
662 cld /* nope, copy forwards */
666 andl $3,%ecx /* any bytes left? */
674 /*****************************************************************************/
675 /* copyout and fubyte family */
676 /*****************************************************************************/
678 * Access user memory from inside the kernel. These routines and possibly
679 * the math- and DOS emulators should be the only places that do this.
681 * We have to access the memory with user's permissions, so use a segment
682 * selector with RPL 3. For writes to user space we have to additionally
683 * check the PTE for write permission, because the 386 does not check
684 * write permissions when we are executing with EPL 0. The 486 does check
685 * this if the WP bit is set in CR0, so we can use a simpler version here.
687 * These routines set curpcb->onfault for the time they execute. When a
688 * protection violation occurs inside the functions, the trap handler
689 * returns to *curpcb->onfault instead of the function.
693 * copyout(from_kernel, to_user, len) - MP SAFE (if not I386_CPU)
699 ENTRY(generic_copyout)
700 movl PCPU(CURPCB),%eax
701 movl $copyout_fault,PCB_ONFAULT(%eax)
708 testl %ebx,%ebx /* anything to do? */
712 * Check explicitly for non-user addresses. If 486 write protection
713 * is being used, this check is essential because we are in kernel
714 * mode so the h/w does not provide any protection against writing
719 * First, prevent address wrapping.
725 * XXX STOP USING VM_MAXUSER_ADDRESS.
726 * It is an end address, not a max, so every time it is used correctly it
727 * looks like there is an off by one error, and of course it caused an off
728 * by one error in several places.
730 cmpl $VM_MAXUSER_ADDRESS,%eax
736 * We have to check each PTE for user write permission.
737 * The checking may cause a page fault, so it is important to set
738 * up everything for return via copyout_fault before here.
740 /* compute number of pages */
745 shrl $IDXSHIFT+2,%ecx
748 /* compute PTE offset for start address */
754 /* check PTE for each page */
755 leal PTmap(%edx),%eax
758 testb $PG_V,PTmap(%eax) /* PTE page must be valid */
761 andb $PG_V|PG_RW|PG_U,%al /* page must be valid and user writable */
762 cmpb $PG_V|PG_RW|PG_U,%al
766 /* simulate a trap */
771 call trapwrite /* trapwrite(addr) */
776 testl %eax,%eax /* if not ok, return EFAULT */
782 jnz 1b /* check next page */
783 #endif /* I386_CPU */
785 /* bcopy(%esi, %edi, %ebx) */
788 #if defined(I586_CPU) && defined(DEV_NPX)
806 movl PCPU(CURPCB),%edx
807 movl %eax,PCB_ONFAULT(%edx)
815 movl PCPU(CURPCB),%edx
816 movl $0,PCB_ONFAULT(%edx)
820 #if defined(I586_CPU) && defined(DEV_NPX)
823 * Duplicated from generic_copyout. Could be done a bit better.
825 movl PCPU(CURPCB),%eax
826 movl $copyout_fault,PCB_ONFAULT(%eax)
833 testl %ebx,%ebx /* anything to do? */
837 * Check explicitly for non-user addresses. If 486 write protection
838 * is being used, this check is essential because we are in kernel
839 * mode so the h/w does not provide any protection against writing
844 * First, prevent address wrapping.
850 * XXX STOP USING VM_MAXUSER_ADDRESS.
851 * It is an end address, not a max, so every time it is used correctly it
852 * looks like there is an off by one error, and of course it caused an off
853 * by one error in several places.
855 cmpl $VM_MAXUSER_ADDRESS,%eax
858 /* bcopy(%esi, %edi, %ebx) */
862 * End of duplicated code.
872 #endif /* I586_CPU && defined(DEV_NPX) */
875 * copyin(from_user, to_kernel, len) - MP SAFE
881 ENTRY(generic_copyin)
882 movl PCPU(CURPCB),%eax
883 movl $copyin_fault,PCB_ONFAULT(%eax)
886 movl 12(%esp),%esi /* caddr_t from */
887 movl 16(%esp),%edi /* caddr_t to */
888 movl 20(%esp),%ecx /* size_t len */
891 * make sure address is valid
896 cmpl $VM_MAXUSER_ADDRESS,%edx
899 #if defined(I586_CPU) && defined(DEV_NPX)
904 shrl $2,%ecx /* copy longword-wise */
909 andb $3,%cl /* copy remaining bytes */
913 #if defined(I586_CPU) && defined(DEV_NPX)
920 movl PCPU(CURPCB),%edx
921 movl %eax,PCB_ONFAULT(%edx)
928 movl PCPU(CURPCB),%edx
929 movl $0,PCB_ONFAULT(%edx)
933 #if defined(I586_CPU) && defined(DEV_NPX)
936 * Duplicated from generic_copyin. Could be done a bit better.
938 movl PCPU(CURPCB),%eax
939 movl $copyin_fault,PCB_ONFAULT(%eax)
942 movl 12(%esp),%esi /* caddr_t from */
943 movl 16(%esp),%edi /* caddr_t to */
944 movl 20(%esp),%ecx /* size_t len */
947 * make sure address is valid
952 cmpl $VM_MAXUSER_ADDRESS,%edx
955 * End of duplicated code.
961 pushl %ebx /* XXX prepare for fastmove_fault */
966 #endif /* I586_CPU && defined(DEV_NPX) */
968 #if defined(I586_CPU) && defined(DEV_NPX)
969 /* fastmove(src, dst, len)
972 len in %ecx XXX changed to on stack for profiling
973 uses %eax and %edx for tmp. storage
975 /* XXX use ENTRY() to get profiling. fastmove() is actually a non-entry. */
979 subl $PCB_SAVEFPU_SIZE+3*4,%esp
985 testl $7,%esi /* check if src addr is multiple of 8 */
988 testl $7,%edi /* check if dst addr is multiple of 8 */
991 /* XXX grab FPU context atomically. */
994 /* if (npxproc != NULL) { */
995 cmpl $0,PCPU(NPXPROC)
997 /* fnsave(&curpcb->pcb_savefpu); */
998 movl PCPU(CURPCB),%eax
999 fnsave PCB_SAVEFPU(%eax)
1000 /* npxproc = NULL; */
1001 movl $0,PCPU(NPXPROC)
1004 /* now we own the FPU. */
1007 * The process' FP state is saved in the pcb, but if we get
1008 * switched, the cpu_switch() will store our FP state in the
1009 * pcb. It should be possible to avoid all the copying for
1010 * this, e.g., by setting a flag to tell cpu_switch() to
1011 * save the state somewhere else.
1013 /* tmp = curpcb->pcb_savefpu; */
1018 movl PCPU(CURPCB),%esi
1019 addl $PCB_SAVEFPU,%esi
1021 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1027 /* stop_emulating(); */
1029 /* npxproc = curproc; */
1030 movl PCPU(CURPROC),%eax
1031 movl %eax,PCPU(NPXPROC)
1032 movl PCPU(CURPCB),%eax
1034 /* XXX end of atomic FPU context grab. */
1037 movl $fastmove_fault,PCB_ONFAULT(%eax)
1094 /* XXX ungrab FPU context atomically. */
1097 /* curpcb->pcb_savefpu = tmp; */
1101 movl PCPU(CURPCB),%edi
1102 addl $PCB_SAVEFPU,%edi
1105 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1112 /* start_emulating(); */
1116 /* npxproc = NULL; */
1117 movl $0,PCPU(NPXPROC)
1119 /* XXX end of atomic FPU context ungrab. */
1124 movl PCPU(CURPCB),%eax
1125 movl $fastmove_tail_fault,PCB_ONFAULT(%eax)
1128 shrl $2,%ecx /* copy longword-wise */
1133 andb $3,%cl /* copy remaining bytes */
1143 /* XXX ungrab FPU context atomically. */
1146 movl PCPU(CURPCB),%edi
1147 addl $PCB_SAVEFPU,%edi
1150 movl $PCB_SAVEFPU_SIZE>>2,%ecx
1157 movl $0,PCPU(NPXPROC)
1159 /* XXX end of atomic FPU context ungrab. */
1162 fastmove_tail_fault:
1169 movl PCPU(CURPCB),%edx
1170 movl $0,PCB_ONFAULT(%edx)
1173 #endif /* I586_CPU && defined(DEV_NPX) */
1176 * fu{byte,sword,word} - MP SAFE
1178 * Fetch a byte (sword, word) from user memory
1181 movl PCPU(CURPCB),%ecx
1182 movl $fusufault,PCB_ONFAULT(%ecx)
1183 movl 4(%esp),%edx /* from */
1185 cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address is valid */
1189 movl $0,PCB_ONFAULT(%ecx)
1193 * These two routines are called from the profiling code, potentially
1194 * at interrupt time. If they fail, that's okay, good things will
1195 * happen later. Fail all the time for now - until the trap code is
1196 * able to deal with this.
1207 movl PCPU(CURPCB),%ecx
1208 movl $fusufault,PCB_ONFAULT(%ecx)
1211 cmpl $VM_MAXUSER_ADDRESS-2,%edx
1215 movl $0,PCB_ONFAULT(%ecx)
1222 movl PCPU(CURPCB),%ecx
1223 movl $fusufault,PCB_ONFAULT(%ecx)
1226 cmpl $VM_MAXUSER_ADDRESS-1,%edx
1230 movl $0,PCB_ONFAULT(%ecx)
1235 movl PCPU(CURPCB),%ecx
1237 movl %eax,PCB_ONFAULT(%ecx)
1242 * su{byte,sword,word} - MP SAFE (if not I386_CPU)
1244 * Write a byte (word, longword) to user memory
1247 movl PCPU(CURPCB),%ecx
1248 movl $fusufault,PCB_ONFAULT(%ecx)
1253 /* XXX - page boundary crossing is still not handled */
1258 leal PTmap(%edx),%ecx
1261 testb $PG_V,PTmap(%ecx) /* PTE page must be valid */
1263 movb PTmap(%edx),%dl
1264 andb $PG_V|PG_RW|PG_U,%dl /* page must be valid and user writable */
1265 cmpb $PG_V|PG_RW|PG_U,%dl
1269 /* simulate a trap */
1272 popl %edx /* remove junk parameter from stack */
1279 cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address validity */
1285 movl PCPU(CURPCB),%ecx
1286 movl %eax,PCB_ONFAULT(%ecx)
1290 * susword - MP SAFE (if not I386_CPU)
1293 movl PCPU(CURPCB),%ecx
1294 movl $fusufault,PCB_ONFAULT(%ecx)
1299 /* XXX - page boundary crossing is still not handled */
1304 leal PTmap(%edx),%ecx
1307 testb $PG_V,PTmap(%ecx) /* PTE page must be valid */
1309 movb PTmap(%edx),%dl
1310 andb $PG_V|PG_RW|PG_U,%dl /* page must be valid and user writable */
1311 cmpb $PG_V|PG_RW|PG_U,%dl
1315 /* simulate a trap */
1318 popl %edx /* remove junk parameter from stack */
1325 cmpl $VM_MAXUSER_ADDRESS-2,%edx /* verify address validity */
1331 movl PCPU(CURPCB),%ecx /* restore trashed register */
1332 movl %eax,PCB_ONFAULT(%ecx)
1336 * su[i]byte - MP SAFE (if not I386_CPU)
1340 movl PCPU(CURPCB),%ecx
1341 movl $fusufault,PCB_ONFAULT(%ecx)
1350 leal PTmap(%edx),%ecx
1353 testb $PG_V,PTmap(%ecx) /* PTE page must be valid */
1355 movb PTmap(%edx),%dl
1356 andb $PG_V|PG_RW|PG_U,%dl /* page must be valid and user writable */
1357 cmpb $PG_V|PG_RW|PG_U,%dl
1361 /* simulate a trap */
1364 popl %edx /* remove junk parameter from stack */
1371 cmpl $VM_MAXUSER_ADDRESS-1,%edx /* verify address validity */
1377 movl PCPU(CURPCB),%ecx /* restore trashed register */
1378 movl %eax,PCB_ONFAULT(%ecx)
1382 * copyinstr(from, to, maxlen, int *lencopied) - MP SAFE
1384 * copy a string from from to to, stop when a 0 character is reached.
1385 * return ENAMETOOLONG if string is longer than maxlen, and
1386 * EFAULT on protection violations. If lencopied is non-zero,
1387 * return the actual length in *lencopied.
1392 movl PCPU(CURPCB),%ecx
1393 movl $cpystrflt,PCB_ONFAULT(%ecx)
1395 movl 12(%esp),%esi /* %esi = from */
1396 movl 16(%esp),%edi /* %edi = to */
1397 movl 20(%esp),%edx /* %edx = maxlen */
1399 movl $VM_MAXUSER_ADDRESS,%eax
1401 /* make sure 'from' is within bounds */
1405 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
1423 /* Success -- 0 byte reached */
1428 /* edx is zero - return ENAMETOOLONG or EFAULT */
1429 cmpl $VM_MAXUSER_ADDRESS,%esi
1432 movl $ENAMETOOLONG,%eax
1439 /* set *lencopied and return %eax */
1440 movl PCPU(CURPCB),%ecx
1441 movl $0,PCB_ONFAULT(%ecx)
1455 * copystr(from, to, maxlen, int *lencopied) - MP SAFE
1461 movl 12(%esp),%esi /* %esi = from */
1462 movl 16(%esp),%edi /* %edi = to */
1463 movl 20(%esp),%edx /* %edx = maxlen */
1474 /* Success -- 0 byte reached */
1479 /* edx is zero -- return ENAMETOOLONG */
1480 movl $ENAMETOOLONG,%eax
1483 /* set *lencopied and return %eax */
1505 cld /* compare forwards */
1524 * Handling of special 386 registers and descriptor tables etc
1526 /* void lgdt(struct region_descriptor *rdp); */
1528 /* reload the descriptor table */
1532 /* flush the prefetch q */
1536 /* reload "stale" selectors */
1545 /* reload code selector by turning return into intersegmental return */
1552 * void lidt(struct region_descriptor *rdp);
1560 * void lldt(u_short sel)
1567 * void ltr(u_short sel)
1573 /* ssdtosd(*ssdp,*sdp) */
1610 /* void load_cr3(caddr_t cr3) */
1612 #ifdef SWTCH_OPTIM_STATS
1613 incl tlb_flush_count
1624 /* void load_cr4(caddr_t cr4) */
1630 /* void load_dr6(u_int dr6) */
1636 /* void reset_dbregs() */
1639 movl %eax,%dr7 /* disable all breapoints first */
1647 /*****************************************************************************/
1648 /* setjump, longjump */
1649 /*****************************************************************************/
1653 movl %ebx,(%eax) /* save ebx */
1654 movl %esp,4(%eax) /* save esp */
1655 movl %ebp,8(%eax) /* save ebp */
1656 movl %esi,12(%eax) /* save esi */
1657 movl %edi,16(%eax) /* save edi */
1658 movl (%esp),%edx /* get rta */
1659 movl %edx,20(%eax) /* save eip */
1660 xorl %eax,%eax /* return(0); */
1665 movl (%eax),%ebx /* restore ebx */
1666 movl 4(%eax),%esp /* restore esp */
1667 movl 8(%eax),%ebp /* restore ebp */
1668 movl 12(%eax),%esi /* restore esi */
1669 movl 16(%eax),%edi /* restore edi */
1670 movl 20(%eax),%edx /* get rta */
1671 movl %edx,(%esp) /* put in return frame */
1672 xorl %eax,%eax /* return(1); */
1677 * Support for BB-profiling (gcc -a). The kernbb program will extract
1678 * the data from the kernel.
1688 NON_GPROF_ENTRY(__bb_init_func)