2 * Copyright (c) 1993 The Regents of the University of California.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * Functions to provide access to special i386 instructions.
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define _MACHINE_CPUFUNC_H_
43 #define readb(va) (*(volatile u_int8_t *) (va))
44 #define readw(va) (*(volatile u_int16_t *) (va))
45 #define readl(va) (*(volatile u_int32_t *) (va))
47 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d))
48 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d))
49 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d))
54 #include <machine/lock.h> /* XXX */
57 #ifdef SWTCH_OPTIM_STATS
58 extern int tlb_flush_count; /* XXX */
64 __asm __volatile("int $3");
72 __asm __volatile("bsfl %0,%0" : "=r" (result) : "0" (mask));
81 __asm __volatile("bsrl %0,%0" : "=r" (result) : "0" (mask));
88 __asm __volatile("cli" : : : "memory");
100 __asm __volatile("sti");
103 #define HAVE_INLINE_FFS
109 * Note that gcc-2's builtin ffs would be used if we didn't declare
110 * this inline or turn off the builtin. The builtin is faster but
111 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
114 return (mask == 0 ? mask : bsfl((u_int)mask) + 1);
117 #define HAVE_INLINE_FLS
122 return (mask == 0 ? mask : bsrl((u_int)mask) + 1);
127 #define inb(port) inbv(port)
128 #define outb(port, data) outbv(port, data)
130 #else /* __GNUC >= 2 */
133 * The following complications are to get around gcc not having a
134 * constraint letter for the range 0..255. We still put "d" in the
135 * constraint because "i" isn't a valid constraint when the port
136 * isn't constant. This only matters for -O0 because otherwise
137 * the non-working version gets optimized away.
139 * Use an expression-statement instead of a conditional expression
140 * because gcc-2.6.0 would promote the operands of the conditional
141 * and produce poor code for "if ((inb(var) & const1) == const2)".
143 * The unnecessary test `(port) < 0x10000' is to generate a warning if
144 * the `port' has type u_short or smaller. Such types are pessimal.
145 * This actually only works for signed types. The range check is
146 * careful to avoid generating warnings.
148 #define inb(port) __extension__ ({ \
150 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \
151 && (port) < 0x10000) \
152 _data = inbc(port); \
154 _data = inbv(port); \
157 #define outb(port, data) ( \
158 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \
159 && (port) < 0x10000 \
160 ? outbc(port, data) : outbv(port, data))
162 static __inline u_char
167 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
172 outbc(u_int port, u_char data)
174 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
177 #endif /* __GNUC <= 2 */
179 static __inline u_char
184 * We use %%dx and not %1 here because i/o is done at %dx and not at
185 * %edx, while gcc generates inferior code (movw instead of movl)
186 * if we tell it to load (u_short) port.
188 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
192 static __inline u_int
197 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
202 insb(u_int port, void *addr, size_t cnt)
204 __asm __volatile("cld; rep; insb"
205 : "=D" (addr), "=c" (cnt)
206 : "0" (addr), "1" (cnt), "d" (port)
211 insw(u_int port, void *addr, size_t cnt)
213 __asm __volatile("cld; rep; insw"
214 : "=D" (addr), "=c" (cnt)
215 : "0" (addr), "1" (cnt), "d" (port)
220 insl(u_int port, void *addr, size_t cnt)
222 __asm __volatile("cld; rep; insl"
223 : "=D" (addr), "=c" (cnt)
224 : "0" (addr), "1" (cnt), "d" (port)
231 __asm __volatile("invd");
234 #if defined(SMP) && defined(_KERNEL)
237 * When using APIC IPI's, invlpg() is not simply the invlpg instruction
238 * (this is a bug) and the inlining cost is prohibitive since the call
239 * executes into the IPI transmission system.
241 void invlpg __P((u_int addr));
242 void invltlb __P((void));
245 cpu_invlpg(void *addr)
247 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
255 * This should be implemented as load_cr3(rcr3()) when load_cr3()
258 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
260 #if defined(SWTCH_OPTIM_STATS)
265 #else /* !(SMP && _KERNEL) */
270 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
278 * This should be implemented as load_cr3(rcr3()) when load_cr3()
281 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
283 #ifdef SWTCH_OPTIM_STATS
288 #endif /* SMP && _KERNEL */
290 static __inline u_short
295 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
299 static __inline u_int
300 loadandclear(volatile u_int *addr)
304 __asm __volatile("xorl %0,%0; xchgl %1,%0"
305 : "=&r" (result) : "m" (*addr));
310 outbv(u_int port, u_char data)
314 * Use an unnecessary assignment to help gcc's register allocator.
315 * This make a large difference for gcc-1.40 and a tiny difference
316 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for
317 * best results. gcc-2.6.0 can't handle this.
320 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
324 outl(u_int port, u_int data)
327 * outl() and outw() aren't used much so we haven't looked at
328 * possible micro-optimizations such as the unnecessary
329 * assignment for them.
331 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
335 outsb(u_int port, const void *addr, size_t cnt)
337 __asm __volatile("cld; rep; outsb"
338 : "=S" (addr), "=c" (cnt)
339 : "0" (addr), "1" (cnt), "d" (port));
343 outsw(u_int port, const void *addr, size_t cnt)
345 __asm __volatile("cld; rep; outsw"
346 : "=S" (addr), "=c" (cnt)
347 : "0" (addr), "1" (cnt), "d" (port));
351 outsl(u_int port, const void *addr, size_t cnt)
353 __asm __volatile("cld; rep; outsl"
354 : "=S" (addr), "=c" (cnt)
355 : "0" (addr), "1" (cnt), "d" (port));
359 outw(u_int port, u_short data)
361 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
364 static __inline u_int
369 __asm __volatile("movl %%cr2,%0" : "=r" (data));
373 static __inline u_int
378 __asm __volatile("pushfl; popl %0" : "=r" (ef));
382 static __inline u_int64_t
387 __asm __volatile(".byte 0x0f, 0x32" : "=A" (rv) : "c" (msr));
391 static __inline u_int64_t
396 __asm __volatile(".byte 0x0f, 0x33" : "=A" (rv) : "c" (pmc));
400 static __inline u_int64_t
405 __asm __volatile(".byte 0x0f, 0x31" : "=A" (rv));
412 __asm __volatile("wbinvd");
416 write_eflags(u_int ef)
418 __asm __volatile("pushl %0; popfl" : : "r" (ef));
422 wrmsr(u_int msr, u_int64_t newval)
424 __asm __volatile(".byte 0x0f, 0x30" : : "A" (newval), "c" (msr));
427 static __inline u_int
431 __asm __volatile("movl %%fs,%0" : "=rm" (sel));
435 static __inline u_int
439 __asm __volatile("movl %%gs,%0" : "=rm" (sel));
446 __asm __volatile("movl %0,%%fs" : : "rm" (sel));
452 __asm __volatile("movl %0,%%gs" : : "rm" (sel));
455 #else /* !__GNUC__ */
457 int breakpoint __P((void));
458 u_int bsfl __P((u_int mask));
459 u_int bsrl __P((u_int mask));
460 void disable_intr __P((void));
461 void enable_intr __P((void));
462 u_char inb __P((u_int port));
463 u_int inl __P((u_int port));
464 void insb __P((u_int port, void *addr, size_t cnt));
465 void insl __P((u_int port, void *addr, size_t cnt));
466 void insw __P((u_int port, void *addr, size_t cnt));
467 void invd __P((void));
468 void invlpg __P((u_int addr));
469 void invltlb __P((void));
470 u_short inw __P((u_int port));
471 u_int loadandclear __P((u_int *addr));
472 void outb __P((u_int port, u_char data));
473 void outl __P((u_int port, u_int data));
474 void outsb __P((u_int port, void *addr, size_t cnt));
475 void outsl __P((u_int port, void *addr, size_t cnt));
476 void outsw __P((u_int port, void *addr, size_t cnt));
477 void outw __P((u_int port, u_short data));
478 u_int rcr2 __P((void));
479 u_int64_t rdmsr __P((u_int msr));
480 u_int64_t rdpmc __P((u_int pmc));
481 u_int64_t rdtsc __P((void));
482 u_int read_eflags __P((void));
483 void wbinvd __P((void));
484 void write_eflags __P((u_int ef));
485 void wrmsr __P((u_int msr, u_int64_t newval));
486 u_int rfs __P((void));
487 u_int rgs __P((void));
488 void load_fs __P((u_int sel));
489 void load_gs __P((u_int sel));
491 #endif /* __GNUC__ */
493 void load_cr0 __P((u_int cr0));
494 void load_cr3 __P((u_int cr3));
495 void load_cr4 __P((u_int cr4));
496 void ltr __P((u_short sel));
497 u_int rcr0 __P((void));
498 u_int rcr3 __P((void));
499 u_int rcr4 __P((void));
501 #endif /* !_MACHINE_CPUFUNC_H_ */