2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2018 The FreeBSD Foundation
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
41 #include <sys/sysctl.h>
42 #include <sys/systm.h>
44 #include <vm/vm_param.h>
45 #include <vm/vm_extern.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_page.h>
50 #if defined(PAE) || defined(PAE_TABLES)
51 #define KCR3 ((u_int)IdlePDPT)
53 #define KCR3 ((u_int)IdlePTD)
56 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
57 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
58 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
59 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
60 int fubyte_fast(volatile const void *base, u_int kcr3);
61 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
62 int fuword16_fast(volatile const void *base, u_int kcr3);
63 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
64 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
65 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
66 int subyte_fast(volatile void *base, int val, u_int kcr3);
67 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
68 int suword16_fast(volatile void *base, int val, u_int kcr3);
69 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
70 int suword_fast(volatile void *base, long val, u_int kcr3);
71 static int (*suword_fast_tramp)(volatile void *, long, u_int);
73 static int fast_copyout = 1;
74 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
79 copyout_init_tramp(void)
82 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
83 (uintptr_t)copyin_fast + setidt_disp);
84 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
85 (uintptr_t)copyout_fast + setidt_disp);
86 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
87 (uintptr_t)fubyte_fast + setidt_disp);
88 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
89 (uintptr_t)fuword16_fast + setidt_disp);
90 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
91 (uintptr_t)fueword_fast + setidt_disp);
92 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
93 (uintptr_t)subyte_fast + setidt_disp);
94 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
95 (uintptr_t)suword16_fast + setidt_disp);
96 suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
97 (uintptr_t)suword_fast + setidt_disp);
101 cp_slow0(vm_offset_t uva, size_t len, bool write,
102 void (*f)(vm_offset_t, void *), void *arg)
111 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
112 MPASS(plen <= nitems(m));
114 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
115 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
121 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
122 (curthread->td_pflags & TDP_NOFAULTING) != 0) {
124 mtx_lock(&pc->pc_copyout_mlock);
125 kaddr = pc->pc_copyout_maddr;
128 sx_xlock(&pc->pc_copyout_slock);
129 kaddr = pc->pc_copyout_saddr;
131 for (i = 0, pte = vtopte(kaddr); i < plen; i++, pte++) {
132 *pte = PG_V | PG_RW | PG_A | PG_M | VM_PAGE_TO_PHYS(m[i]) |
133 pmap_cache_bits(pmap_page_get_memattr(m[i]), FALSE);
134 invlpg(kaddr + ptoa(i));
136 kaddr += uva - trunc_page(uva);
140 sx_xunlock(&pc->pc_copyout_slock);
142 mtx_unlock(&pc->pc_copyout_mlock);
143 vm_page_unhold_pages(m, plen);
147 struct copyinstr_arg0 {
155 copyinstr_slow0(vm_offset_t kva, void *arg)
157 struct copyinstr_arg0 *ca;
161 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
162 while (ca->alen < ca->len && !ca->end) {
163 c = *(char *)(kva + ca->alen);
173 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
175 struct copyinstr_arg0 ca;
182 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
183 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
184 ca.len = round_page(uc) - uc;
187 if (plen + ca.len > maxlen)
188 ca.len = maxlen - plen;
190 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
195 if (!ca.end && plen == maxlen && error == 0)
196 error = ENAMETOOLONG;
197 if (lencopied != NULL)
208 copyin_slow0(vm_offset_t kva, void *arg)
210 struct copyin_arg0 *ca;
213 bcopy((void *)kva, (void *)ca->kc, ca->len);
217 copyin(const void *udaddr, void *kaddr, size_t len)
219 struct copyin_arg0 ca;
223 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
224 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
226 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
227 copyin_fast_tramp(udaddr, kaddr, len, KCR3) == 0))
229 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
230 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
231 ca.len = round_page(uc) - uc;
234 if (plen + ca.len > len)
236 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
243 copyout_slow0(vm_offset_t kva, void *arg)
245 struct copyin_arg0 *ca;
248 bcopy((void *)ca->kc, (void *)kva, ca->len);
252 copyout(const void *kaddr, void *udaddr, size_t len)
254 struct copyin_arg0 ca;
258 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
259 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
261 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
262 copyout_fast_tramp(kaddr, udaddr, len, KCR3) == 0))
264 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
265 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
266 ca.len = round_page(uc) - uc;
269 if (plen + ca.len > len)
271 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
278 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
283 fubyte_slow0(vm_offset_t kva, void *arg)
286 *(int *)arg = *(u_char *)kva;
290 fubyte(volatile const void *base)
294 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
295 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
298 res = fubyte_fast_tramp(base, KCR3);
302 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
309 fuword16_slow0(vm_offset_t kva, void *arg)
312 *(int *)arg = *(uint16_t *)kva;
316 fuword16(volatile const void *base)
320 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
321 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
324 res = fuword16_fast_tramp(base, KCR3);
328 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
329 fuword16_slow0, &res) != 0)
335 fueword_slow0(vm_offset_t kva, void *arg)
338 *(uint32_t *)arg = *(uint32_t *)kva;
342 fueword(volatile const void *base, long *val)
346 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
347 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
350 if (fueword_fast_tramp(base, val, KCR3) == 0)
353 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
361 fueword32(volatile const void *base, int32_t *val)
364 return (fueword(base, (long *)val));
368 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
372 subyte_slow0(vm_offset_t kva, void *arg)
375 *(u_char *)kva = *(int *)arg;
379 subyte(volatile void *base, int byte)
382 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
383 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
385 if (fast_copyout && subyte_fast_tramp(base, byte, KCR3) == 0)
387 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
388 &byte) != 0 ? -1 : 0);
392 suword16_slow0(vm_offset_t kva, void *arg)
395 *(int *)kva = *(uint16_t *)arg;
399 suword16(volatile void *base, int word)
402 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
403 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
405 if (fast_copyout && suword16_fast_tramp(base, word, KCR3) == 0)
407 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
408 suword16_slow0, &word) != 0 ? -1 : 0);
412 suword_slow0(vm_offset_t kva, void *arg)
415 *(int *)kva = *(uint32_t *)arg;
419 suword(volatile void *base, long word)
422 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
423 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
425 if (fast_copyout && suword_fast_tramp(base, word, KCR3) == 0)
427 return (cp_slow0((vm_offset_t)base, sizeof(long), true,
428 suword_slow0, &word) != 0 ? -1 : 0);
432 suword32(volatile void *base, int32_t word)
435 return (suword(base, word));
438 struct casueword_arg0 {
444 casueword_slow0(vm_offset_t kva, void *arg)
446 struct casueword_arg0 *ca;
449 atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval);
453 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
456 struct casueword_arg0 ca;
461 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
462 casueword_slow0, &ca);
464 *oldvalp = ca.oldval;
471 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
473 struct casueword_arg0 ca;
478 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
479 casueword_slow0, &ca);
481 *oldvalp = ca.oldval;