2 * Copyright (c) 1993 The Regents of the University of California.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <machine/asmacros.h>
33 #include <machine/cputypes.h>
34 #include <machine/pmap.h>
35 #include <machine/specialreg.h>
45 * void bzero(void *buf, u_int len)
71 * The loop takes 14 bytes. Ensure that it doesn't cross a 16-byte
136 /* fillw(pat, base, cnt) */
149 * memmove(dst, src, cnt) (return dst)
150 * bcopy(src, dst, cnt)
151 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
174 cmpl %ecx,%eax /* overlapping && src < dst? */
177 shrl $2,%ecx /* copy by 32-bit words */
181 andl $3,%ecx /* any bytes left? */
186 movl 8(%ebp),%eax /* return dst for memmove */
192 addl %ecx,%edi /* copy backwards */
196 andl $3,%ecx /* any fractional bytes? */
200 movl 16(%ebp),%ecx /* copy remainder by 32-bit words */
209 movl 8(%ebp),%eax /* return dst for memmove */
215 * Note: memcpy does not support overlapping copies
224 shrl $2,%ecx /* copy by 32-bit words */
228 andl $3,%ecx /* any bytes left? */
262 * Handling of special 386 registers and descriptor tables etc
264 /* void lgdt(struct region_descriptor *rdp); */
266 /* reload the descriptor table */
270 /* flush the prefetch q */
274 /* reload "stale" selectors */
283 /* reload code selector by turning return into intersegmental return */
291 /* ssdtosd(*ssdp,*sdp) */
313 /* void reset_dbregs() */
316 movl %eax,%dr7 /* disable all breakpoints first */
325 /*****************************************************************************/
326 /* setjump, longjump */
327 /*****************************************************************************/
331 movl %ebx,(%eax) /* save ebx */
332 movl %esp,4(%eax) /* save esp */
333 movl %ebp,8(%eax) /* save ebp */
334 movl %esi,12(%eax) /* save esi */
335 movl %edi,16(%eax) /* save edi */
336 movl (%esp),%edx /* get rta */
337 movl %edx,20(%eax) /* save eip */
338 xorl %eax,%eax /* return(0); */
344 movl (%eax),%ebx /* restore ebx */
345 movl 4(%eax),%esp /* restore esp */
346 movl 8(%eax),%ebp /* restore ebp */
347 movl 12(%eax),%esi /* restore esi */
348 movl 16(%eax),%edi /* restore edi */
349 movl 20(%eax),%edx /* get rta */
350 movl %edx,(%esp) /* put in return frame */
351 xorl %eax,%eax /* return(1); */
357 * Support for reading MSRs in the safe manner. (Instead of panic on #gp,
361 /* int rdmsr_safe(u_int msr, uint64_t *data) */
362 movl PCPU(CURPCB),%ecx
363 movl $msr_onfault,PCB_ONFAULT(%ecx)
372 movl PCPU(CURPCB),%ecx
373 movl %eax,PCB_ONFAULT(%ecx)
378 * Support for writing MSRs in the safe manner. (Instead of panic on #gp,
382 /* int wrmsr_safe(u_int msr, uint64_t data) */
383 movl PCPU(CURPCB),%ecx
384 movl $msr_onfault,PCB_ONFAULT(%ecx)
392 movl PCPU(CURPCB),%ecx
393 movl %eax,PCB_ONFAULT(%ecx)
398 * MSR operations fault handler
402 movl PCPU(CURPCB),%ecx
403 movl $0,PCB_ONFAULT(%ecx)
408 .macro rsb_seq_label l
411 .macro rsb_call_label l
429 ENTRY(handle_ibrs_entry)
430 cmpb $0,hw_ibrs_ibpb_active
432 movl $MSR_IA32_SPEC_CTRL,%ecx
434 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
435 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
437 movb $1,PCPU(IBPB_SET)
439 * i386 does not implement SMEP.
442 END(handle_ibrs_entry)
444 ENTRY(handle_ibrs_exit)
445 cmpb $0,PCPU(IBPB_SET)
447 movl $MSR_IA32_SPEC_CTRL,%ecx
449 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
450 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
452 movb $0,PCPU(IBPB_SET)
454 END(handle_ibrs_exit)
456 ENTRY(mds_handler_void)
458 END(mds_handler_void)
460 ENTRY(mds_handler_verw)
466 END(mds_handler_verw)
468 ENTRY(mds_handler_ivb)
473 1: movl PCPU(MDS_BUF), %edx
474 movdqa %xmm0, PCPU(MDS_TMP)
483 2: movntdq %xmm0, (%edx)
489 movdqa PCPU(MDS_TMP),%xmm0
496 ENTRY(mds_handler_bdw)
501 1: movl PCPU(MDS_BUF), %ebx
502 movdqa %xmm0, PCPU(MDS_TMP)
508 2: movntdq %xmm0, (%ebx)
517 movdqa PCPU(MDS_TMP),%xmm0
524 ENTRY(mds_handler_skl_sse)
529 1: movl PCPU(MDS_BUF), %edi
530 movl PCPU(MDS_BUF64), %edx
531 movdqa %xmm0, PCPU(MDS_TMP)
538 2: clflushopt 5376(%edi, %eax, 8)
548 movdqa PCPU(MDS_TMP), %xmm0
553 END(mds_handler_skl_sse)
555 ENTRY(mds_handler_skl_avx)
560 1: movl PCPU(MDS_BUF), %edi
561 movl PCPU(MDS_BUF64), %edx
562 vmovdqa %ymm0, PCPU(MDS_TMP)
563 vpxor %ymm0, %ymm0, %ymm0
566 vorpd (%edx), %ymm0, %ymm0
567 vorpd (%edx), %ymm0, %ymm0
569 2: clflushopt 5376(%edi, %eax, 8)
579 vmovdqa PCPU(MDS_TMP), %ymm0
584 END(mds_handler_skl_avx)
586 ENTRY(mds_handler_skl_avx512)
591 1: movl PCPU(MDS_BUF), %edi
592 movl PCPU(MDS_BUF64), %edx
593 vmovdqa64 %zmm0, PCPU(MDS_TMP)
594 vpxord %zmm0, %zmm0, %zmm0
597 vorpd (%edx), %zmm0, %zmm0
598 vorpd (%edx), %zmm0, %zmm0
600 2: clflushopt 5376(%edi, %eax, 8)
610 vmovdqa64 PCPU(MDS_TMP), %zmm0
615 END(mds_handler_skl_avx512)
617 ENTRY(mds_handler_silvermont)
622 1: movl PCPU(MDS_BUF), %edx
623 movdqa %xmm0, PCPU(MDS_TMP)
627 2: movntdq %xmm0, (%edx)
633 movdqa PCPU(MDS_TMP),%xmm0
638 END(mds_handler_silvermont)