2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND BSD-4-Clause
4 * Copyright (C) 2002 Benno Rice
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * Copyright (C) 1993 Wolfgang Solfrank.
29 * Copyright (C) 1993 TooLs GmbH.
30 * All rights reserved.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by TooLs GmbH.
43 * 4. The name of TooLs GmbH may not be used to endorse or promote products
44 * derived from this software without specific prior written permission.
46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 #include <sys/cdefs.h>
59 __FBSDID("$FreeBSD$");
61 #include <sys/param.h>
63 #include <sys/mutex.h>
64 #include <sys/systm.h>
69 #include <vm/vm_extern.h>
70 #include <vm/vm_map.h>
72 #include <machine/mmuvar.h>
73 #include <machine/pcb.h>
74 #include <machine/vmparam.h>
75 #include <machine/ifunc.h>
78 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best
79 * option based on the PMAP in use.
81 * There are two options for copy functions on powerpc64:
82 * - 'remap' copies, which remap userspace segments into kernel space for
83 * copying. This is used by the 'oea64' pmap.
84 * - 'direct' copies, which copy directly from userspace. This does not require
85 * remapping user segments into kernel. This is used by the 'radix' pmap for
88 * Book-E does not use the C 'remap' functions, opting instead to use the
89 * 'direct' copies, directly, avoiding the IFUNC overhead.
91 * On 32-bit AIM these functions bypass the IFUNC machinery for performance.
94 int subyte_remap(volatile void *addr, int byte);
95 int subyte_direct(volatile void *addr, int byte);
96 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done);
97 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done);
98 int copyout_remap(const void *kaddr, void *udaddr, size_t len);
99 int copyout_direct(const void *kaddr, void *udaddr, size_t len);
100 int copyin_remap(const void *uaddr, void *kaddr, size_t len);
101 int copyin_direct(const void *uaddr, void *kaddr, size_t len);
102 int suword16_remap(volatile void *addr, int word);
103 int suword16_direct(volatile void *addr, int word);
104 int suword32_remap(volatile void *addr, int word);
105 int suword32_direct(volatile void *addr, int word);
106 int suword_remap(volatile void *addr, long word);
107 int suword_direct(volatile void *addr, long word);
108 int suword64_remap(volatile void *addr, int64_t word);
109 int suword64_direct(volatile void *addr, int64_t word);
110 int fubyte_remap(volatile const void *addr);
111 int fubyte_direct(volatile const void *addr);
112 int fuword16_remap(volatile const void *addr);
113 int fuword16_direct(volatile const void *addr);
114 int fueword32_remap(volatile const void *addr, int32_t *val);
115 int fueword32_direct(volatile const void *addr, int32_t *val);
116 int fueword64_remap(volatile const void *addr, int64_t *val);
117 int fueword64_direct(volatile const void *addr, int64_t *val);
118 int fueword_remap(volatile const void *addr, long *val);
119 int fueword_direct(volatile const void *addr, long *val);
120 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
122 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
124 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp,
126 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp,
130 * The IFUNC resolver determines the copy based on whether the PMAP
131 * implementation includes a pmap_map_user_ptr function.
133 #define DEFINE_COPY_FUNC(ret, func, args) \
134 DEFINE_IFUNC(, ret, func, args) \
136 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \
137 func##_remap : func##_direct); \
139 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int))
140 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *))
141 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t))
142 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t))
143 DEFINE_COPY_FUNC(int, suword, (volatile void *, long))
144 DEFINE_COPY_FUNC(int, suword16, (volatile void *, int))
145 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int))
146 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t))
147 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *))
148 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *))
149 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *))
150 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *))
151 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *))
152 DEFINE_COPY_FUNC(int, casueword32,
153 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t))
154 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long))
156 #define REMAP(x) x##_remap
162 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len)
172 pm = &td->td_proc->p_vmspace->vm_pmap;
174 td->td_pcb->pcb_onfault = &env;
176 td->td_pcb->pcb_onfault = NULL;
184 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
185 td->td_pcb->pcb_onfault = NULL;
196 td->td_pcb->pcb_onfault = NULL;
201 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len)
211 pm = &td->td_proc->p_vmspace->vm_pmap;
213 td->td_pcb->pcb_onfault = &env;
215 td->td_pcb->pcb_onfault = NULL;
223 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
224 td->td_pcb->pcb_onfault = NULL;
235 td->td_pcb->pcb_onfault = NULL;
240 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done)
251 pm = &td->td_proc->p_vmspace->vm_pmap;
256 td->td_pcb->pcb_onfault = &env;
266 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
271 for (i = 0; len > 0 && i < l; i++, t++, len--) {
272 if ((*kp++ = *p++) == 0) {
283 td->td_pcb->pcb_onfault = NULL;
293 REMAP(subyte)(volatile void *addr, int byte)
301 pm = &td->td_proc->p_vmspace->vm_pmap;
303 td->td_pcb->pcb_onfault = &env;
305 td->td_pcb->pcb_onfault = NULL;
309 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
310 td->td_pcb->pcb_onfault = NULL;
316 td->td_pcb->pcb_onfault = NULL;
321 REMAP(suword16)(volatile void *addr, int word)
329 pm = &td->td_proc->p_vmspace->vm_pmap;
331 td->td_pcb->pcb_onfault = &env;
333 td->td_pcb->pcb_onfault = NULL;
337 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
338 td->td_pcb->pcb_onfault = NULL;
344 td->td_pcb->pcb_onfault = NULL;
350 REMAP(suword32)(volatile void *addr, int word)
358 pm = &td->td_proc->p_vmspace->vm_pmap;
360 td->td_pcb->pcb_onfault = &env;
362 td->td_pcb->pcb_onfault = NULL;
366 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
367 td->td_pcb->pcb_onfault = NULL;
373 td->td_pcb->pcb_onfault = NULL;
378 REMAP(suword32)(volatile void *addr, int32_t word)
380 REMAP( return (suword)(addr, (long)word));
385 REMAP(suword)(volatile void *addr, long word)
393 pm = &td->td_proc->p_vmspace->vm_pmap;
395 td->td_pcb->pcb_onfault = &env;
397 td->td_pcb->pcb_onfault = NULL;
401 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
402 td->td_pcb->pcb_onfault = NULL;
408 td->td_pcb->pcb_onfault = NULL;
414 REMAP(suword64)(volatile void *addr, int64_t word)
416 return (REMAP(suword)(addr, (long)word));
421 REMAP(fubyte)(volatile const void *addr)
430 pm = &td->td_proc->p_vmspace->vm_pmap;
432 td->td_pcb->pcb_onfault = &env;
434 td->td_pcb->pcb_onfault = NULL;
438 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
439 td->td_pcb->pcb_onfault = NULL;
445 td->td_pcb->pcb_onfault = NULL;
450 REMAP(fuword16)(volatile const void *addr)
458 pm = &td->td_proc->p_vmspace->vm_pmap;
460 td->td_pcb->pcb_onfault = &env;
462 td->td_pcb->pcb_onfault = NULL;
466 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
467 td->td_pcb->pcb_onfault = NULL;
473 td->td_pcb->pcb_onfault = NULL;
478 REMAP(fueword32)(volatile const void *addr, int32_t *val)
486 pm = &td->td_proc->p_vmspace->vm_pmap;
488 td->td_pcb->pcb_onfault = &env;
490 td->td_pcb->pcb_onfault = NULL;
494 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
495 td->td_pcb->pcb_onfault = NULL;
501 td->td_pcb->pcb_onfault = NULL;
507 REMAP(fueword64)(volatile const void *addr, int64_t *val)
515 pm = &td->td_proc->p_vmspace->vm_pmap;
517 td->td_pcb->pcb_onfault = &env;
519 td->td_pcb->pcb_onfault = NULL;
523 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
524 td->td_pcb->pcb_onfault = NULL;
530 td->td_pcb->pcb_onfault = NULL;
536 REMAP(fueword)(volatile const void *addr, long *val)
544 pm = &td->td_proc->p_vmspace->vm_pmap;
546 td->td_pcb->pcb_onfault = &env;
548 td->td_pcb->pcb_onfault = NULL;
552 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
553 td->td_pcb->pcb_onfault = NULL;
559 td->td_pcb->pcb_onfault = NULL;
564 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
574 pm = &td->td_proc->p_vmspace->vm_pmap;
576 td->td_pcb->pcb_onfault = &env;
578 td->td_pcb->pcb_onfault = NULL;
582 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
584 td->td_pcb->pcb_onfault = NULL;
590 "lwarx %0, 0, %3\n\t" /* load old value */
591 "cmplw %4, %0\n\t" /* compare */
592 "bne 1f\n\t" /* exit if not equal */
593 "stwcx. %5, 0, %3\n\t" /* attempt to store */
594 "bne- 2f\n\t" /* if failed */
595 "b 3f\n\t" /* we've succeeded */
597 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
600 : "=&r" (val), "=m" (*p), "+&r" (res)
601 : "r" (p), "r" (old), "r" (new), "m" (*p)
604 td->td_pcb->pcb_onfault = NULL;
610 #ifndef __powerpc64__
612 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
615 return (casueword32((volatile uint32_t *)addr, old,
616 (uint32_t *)oldvalp, new));
620 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
629 pm = &td->td_proc->p_vmspace->vm_pmap;
631 td->td_pcb->pcb_onfault = &env;
633 td->td_pcb->pcb_onfault = NULL;
637 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
639 td->td_pcb->pcb_onfault = NULL;
645 "ldarx %0, 0, %3\n\t" /* load old value */
646 "cmpld %4, %0\n\t" /* compare */
647 "bne 1f\n\t" /* exit if not equal */
648 "stdcx. %5, 0, %3\n\t" /* attempt to store */
649 "bne- 2f\n\t" /* if failed */
650 "b 3f\n\t" /* we've succeeded */
652 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
655 : "=&r" (val), "=m" (*p), "+&r" (res)
656 : "r" (p), "r" (old), "r" (new), "m" (*p)
659 td->td_pcb->pcb_onfault = NULL;