2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND BSD-4-Clause
4 * Copyright (C) 2002 Benno Rice
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * Copyright (C) 1993 Wolfgang Solfrank.
29 * Copyright (C) 1993 TooLs GmbH.
30 * All rights reserved.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by TooLs GmbH.
43 * 4. The name of TooLs GmbH may not be used to endorse or promote products
44 * derived from this software without specific prior written permission.
46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 #include <sys/cdefs.h>
59 __FBSDID("$FreeBSD$");
61 #include <sys/param.h>
63 #include <sys/mutex.h>
64 #include <sys/systm.h>
69 #include <vm/vm_extern.h>
70 #include <vm/vm_map.h>
72 #include <machine/mmuvar.h>
73 #include <machine/pcb.h>
74 #include <machine/vmparam.h>
75 #include <machine/ifunc.h>
78 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best
79 * option based on the PMAP in use.
81 * There are two options for copy functions on powerpc64:
82 * - 'remap' copies, which remap userspace segments into kernel space for
83 * copying. This is used by the 'oea64' pmap.
84 * - 'direct' copies, which copy directly from userspace. This does not require
85 * remapping user segments into kernel. This is used by the 'radix' pmap for
88 * Book-E does not use the C 'remap' functions, opting instead to use the
89 * 'direct' copies, directly, avoiding the IFUNC overhead.
91 * On 32-bit AIM these functions bypass the IFUNC machinery for performance.
94 int subyte_remap(volatile void *addr, int byte);
95 int subyte_direct(volatile void *addr, int byte);
96 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done);
97 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done);
98 int copyout_remap(const void *kaddr, void *udaddr, size_t len);
99 int copyout_direct(const void *kaddr, void *udaddr, size_t len);
100 int copyin_remap(const void *uaddr, void *kaddr, size_t len);
101 int copyin_direct(const void *uaddr, void *kaddr, size_t len);
102 int suword32_remap(volatile void *addr, int word);
103 int suword32_direct(volatile void *addr, int word);
104 int suword_remap(volatile void *addr, long word);
105 int suword_direct(volatile void *addr, long word);
106 int suword64_remap(volatile void *addr, int64_t word);
107 int suword64_direct(volatile void *addr, int64_t word);
108 int fubyte_remap(volatile const void *addr);
109 int fubyte_direct(volatile const void *addr);
110 int fuword16_remap(volatile const void *addr);
111 int fuword16_direct(volatile const void *addr);
112 int fueword32_remap(volatile const void *addr, int32_t *val);
113 int fueword32_direct(volatile const void *addr, int32_t *val);
114 int fueword64_remap(volatile const void *addr, int64_t *val);
115 int fueword64_direct(volatile const void *addr, int64_t *val);
116 int fueword_remap(volatile const void *addr, long *val);
117 int fueword_direct(volatile const void *addr, long *val);
118 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
120 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
122 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp,
124 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp,
128 * The IFUNC resolver determines the copy based on whether the PMAP
129 * implementation includes a pmap_map_user_ptr function.
131 #define DEFINE_COPY_FUNC(ret, func, args) \
132 DEFINE_IFUNC(, ret, func, args) \
134 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \
135 func##_remap : func##_direct); \
137 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int))
138 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *))
139 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t))
140 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t))
141 DEFINE_COPY_FUNC(int, suword, (volatile void *, long))
142 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int))
143 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t))
144 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *))
145 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *))
146 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *))
147 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *))
148 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *))
149 DEFINE_COPY_FUNC(int, casueword32,
150 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t))
151 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long))
153 #define REMAP(x) x##_remap
159 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len)
169 pm = &td->td_proc->p_vmspace->vm_pmap;
171 td->td_pcb->pcb_onfault = &env;
173 td->td_pcb->pcb_onfault = NULL;
181 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
182 td->td_pcb->pcb_onfault = NULL;
193 td->td_pcb->pcb_onfault = NULL;
198 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len)
208 pm = &td->td_proc->p_vmspace->vm_pmap;
210 td->td_pcb->pcb_onfault = &env;
212 td->td_pcb->pcb_onfault = NULL;
220 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
221 td->td_pcb->pcb_onfault = NULL;
232 td->td_pcb->pcb_onfault = NULL;
237 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done)
249 for (l = 0; len-- > 0; l++) {
250 if ((c = fubyte(up++)) < 0) {
270 REMAP(subyte)(volatile void *addr, int byte)
278 pm = &td->td_proc->p_vmspace->vm_pmap;
280 td->td_pcb->pcb_onfault = &env;
282 td->td_pcb->pcb_onfault = NULL;
286 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
287 td->td_pcb->pcb_onfault = NULL;
293 td->td_pcb->pcb_onfault = NULL;
299 REMAP(suword32)(volatile void *addr, int word)
307 pm = &td->td_proc->p_vmspace->vm_pmap;
309 td->td_pcb->pcb_onfault = &env;
311 td->td_pcb->pcb_onfault = NULL;
315 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
316 td->td_pcb->pcb_onfault = NULL;
322 td->td_pcb->pcb_onfault = NULL;
327 REMAP(suword32)(volatile void *addr, int32_t word)
329 REMAP( return (suword)(addr, (long)word));
334 REMAP(suword)(volatile void *addr, long word)
342 pm = &td->td_proc->p_vmspace->vm_pmap;
344 td->td_pcb->pcb_onfault = &env;
346 td->td_pcb->pcb_onfault = NULL;
350 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
351 td->td_pcb->pcb_onfault = NULL;
357 td->td_pcb->pcb_onfault = NULL;
363 REMAP(suword64)(volatile void *addr, int64_t word)
365 return (REMAP(suword)(addr, (long)word));
370 REMAP(fubyte)(volatile const void *addr)
379 pm = &td->td_proc->p_vmspace->vm_pmap;
381 td->td_pcb->pcb_onfault = &env;
383 td->td_pcb->pcb_onfault = NULL;
387 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
388 td->td_pcb->pcb_onfault = NULL;
394 td->td_pcb->pcb_onfault = NULL;
399 REMAP(fuword16)(volatile const void *addr)
407 pm = &td->td_proc->p_vmspace->vm_pmap;
409 td->td_pcb->pcb_onfault = &env;
411 td->td_pcb->pcb_onfault = NULL;
415 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
416 td->td_pcb->pcb_onfault = NULL;
422 td->td_pcb->pcb_onfault = NULL;
427 REMAP(fueword32)(volatile const void *addr, int32_t *val)
435 pm = &td->td_proc->p_vmspace->vm_pmap;
437 td->td_pcb->pcb_onfault = &env;
439 td->td_pcb->pcb_onfault = NULL;
443 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
444 td->td_pcb->pcb_onfault = NULL;
450 td->td_pcb->pcb_onfault = NULL;
456 REMAP(fueword64)(volatile const void *addr, int64_t *val)
464 pm = &td->td_proc->p_vmspace->vm_pmap;
466 td->td_pcb->pcb_onfault = &env;
468 td->td_pcb->pcb_onfault = NULL;
472 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
473 td->td_pcb->pcb_onfault = NULL;
479 td->td_pcb->pcb_onfault = NULL;
485 REMAP(fueword)(volatile const void *addr, long *val)
493 pm = &td->td_proc->p_vmspace->vm_pmap;
495 td->td_pcb->pcb_onfault = &env;
497 td->td_pcb->pcb_onfault = NULL;
501 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
502 td->td_pcb->pcb_onfault = NULL;
508 td->td_pcb->pcb_onfault = NULL;
513 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
523 pm = &td->td_proc->p_vmspace->vm_pmap;
525 td->td_pcb->pcb_onfault = &env;
527 td->td_pcb->pcb_onfault = NULL;
531 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
533 td->td_pcb->pcb_onfault = NULL;
539 "lwarx %0, 0, %3\n\t" /* load old value */
540 "cmplw %4, %0\n\t" /* compare */
541 "bne 1f\n\t" /* exit if not equal */
542 "stwcx. %5, 0, %3\n\t" /* attempt to store */
543 "bne- 2f\n\t" /* if failed */
544 "b 3f\n\t" /* we've succeeded */
546 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
549 : "=&r" (val), "=m" (*p), "+&r" (res)
550 : "r" (p), "r" (old), "r" (new), "m" (*p)
553 td->td_pcb->pcb_onfault = NULL;
559 #ifndef __powerpc64__
561 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
564 return (casueword32((volatile uint32_t *)addr, old,
565 (uint32_t *)oldvalp, new));
569 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
578 pm = &td->td_proc->p_vmspace->vm_pmap;
580 td->td_pcb->pcb_onfault = &env;
582 td->td_pcb->pcb_onfault = NULL;
586 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
588 td->td_pcb->pcb_onfault = NULL;
594 "ldarx %0, 0, %3\n\t" /* load old value */
595 "cmpld %4, %0\n\t" /* compare */
596 "bne 1f\n\t" /* exit if not equal */
597 "stdcx. %5, 0, %3\n\t" /* attempt to store */
598 "bne- 2f\n\t" /* if failed */
599 "b 3f\n\t" /* we've succeeded */
601 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
604 : "=&r" (val), "=m" (*p), "+&r" (res)
605 : "r" (p), "r" (old), "r" (new), "m" (*p)
608 td->td_pcb->pcb_onfault = NULL;