2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2018 The FreeBSD Foundation
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
37 #include <sys/mutex.h>
40 #include <sys/sched.h>
41 #include <sys/sysctl.h>
42 #include <sys/systm.h>
44 #include <vm/vm_param.h>
45 #include <vm/vm_extern.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_page.h>
50 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
51 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
52 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
53 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
54 int fubyte_fast(volatile const void *base, u_int kcr3);
55 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
56 int fuword16_fast(volatile const void *base, u_int kcr3);
57 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
58 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
59 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
60 int subyte_fast(volatile void *base, int val, u_int kcr3);
61 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
62 int suword16_fast(volatile void *base, int val, u_int kcr3);
63 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
64 int suword_fast(volatile void *base, long val, u_int kcr3);
65 static int (*suword_fast_tramp)(volatile void *, long, u_int);
67 static int fast_copyout = 1;
68 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
73 copyout_init_tramp(void)
76 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
77 (uintptr_t)copyin_fast + setidt_disp);
78 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
79 (uintptr_t)copyout_fast + setidt_disp);
80 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
81 (uintptr_t)fubyte_fast + setidt_disp);
82 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
83 (uintptr_t)fuword16_fast + setidt_disp);
84 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
85 (uintptr_t)fueword_fast + setidt_disp);
86 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
87 (uintptr_t)subyte_fast + setidt_disp);
88 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
89 (uintptr_t)suword16_fast + setidt_disp);
90 suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
91 (uintptr_t)suword_fast + setidt_disp);
95 cp_slow0(vm_offset_t uva, size_t len, bool write,
96 void (*f)(vm_offset_t, void *), void *arg)
104 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
105 MPASS(plen <= nitems(m));
107 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
108 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
114 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
115 (curthread->td_pflags & TDP_NOFAULTING) != 0) {
117 mtx_lock(&pc->pc_copyout_mlock);
118 kaddr = pc->pc_copyout_maddr;
121 sx_xlock(&pc->pc_copyout_slock);
122 kaddr = pc->pc_copyout_saddr;
124 pmap_cp_slow0_map(kaddr, plen, m);
125 kaddr += uva - trunc_page(uva);
129 sx_xunlock(&pc->pc_copyout_slock);
131 mtx_unlock(&pc->pc_copyout_mlock);
132 vm_page_unhold_pages(m, plen);
136 struct copyinstr_arg0 {
144 copyinstr_slow0(vm_offset_t kva, void *arg)
146 struct copyinstr_arg0 *ca;
150 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
151 while (ca->alen < ca->len && !ca->end) {
152 c = *(char *)(kva + ca->alen);
162 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
164 struct copyinstr_arg0 ca;
171 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
172 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
173 ca.len = round_page(uc) - uc;
176 if (plen + ca.len > maxlen)
177 ca.len = maxlen - plen;
179 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
184 if (!ca.end && plen == maxlen && error == 0)
185 error = ENAMETOOLONG;
186 if (lencopied != NULL)
197 copyin_slow0(vm_offset_t kva, void *arg)
199 struct copyin_arg0 *ca;
202 bcopy((void *)kva, (void *)ca->kc, ca->len);
206 copyin(const void *udaddr, void *kaddr, size_t len)
208 struct copyin_arg0 ca;
212 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
213 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
215 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
216 copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0))
218 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
219 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
220 ca.len = round_page(uc) - uc;
223 if (plen + ca.len > len)
225 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
232 copyout_slow0(vm_offset_t kva, void *arg)
234 struct copyin_arg0 *ca;
237 bcopy((void *)ca->kc, (void *)kva, ca->len);
241 copyout(const void *kaddr, void *udaddr, size_t len)
243 struct copyin_arg0 ca;
247 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
248 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
250 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
251 copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0))
253 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
254 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
255 ca.len = round_page(uc) - uc;
258 if (plen + ca.len > len)
260 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
267 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
272 fubyte_slow0(vm_offset_t kva, void *arg)
275 *(int *)arg = *(u_char *)kva;
279 fubyte(volatile const void *base)
283 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
284 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
287 res = fubyte_fast_tramp(base, pmap_get_kcr3());
291 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
298 fuword16_slow0(vm_offset_t kva, void *arg)
301 *(int *)arg = *(uint16_t *)kva;
305 fuword16(volatile const void *base)
309 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
310 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
313 res = fuword16_fast_tramp(base, pmap_get_kcr3());
317 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
318 fuword16_slow0, &res) != 0)
324 fueword_slow0(vm_offset_t kva, void *arg)
327 *(uint32_t *)arg = *(uint32_t *)kva;
331 fueword(volatile const void *base, long *val)
335 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
336 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
339 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0)
342 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
350 fueword32(volatile const void *base, int32_t *val)
353 return (fueword(base, (long *)val));
357 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
361 subyte_slow0(vm_offset_t kva, void *arg)
364 *(u_char *)kva = *(int *)arg;
368 subyte(volatile void *base, int byte)
371 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
372 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
374 if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0)
376 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
377 &byte) != 0 ? -1 : 0);
381 suword16_slow0(vm_offset_t kva, void *arg)
384 *(int *)kva = *(uint16_t *)arg;
388 suword16(volatile void *base, int word)
391 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
392 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
394 if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3())
397 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
398 suword16_slow0, &word) != 0 ? -1 : 0);
402 suword_slow0(vm_offset_t kva, void *arg)
405 *(int *)kva = *(uint32_t *)arg;
409 suword(volatile void *base, long word)
412 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
413 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
415 if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0)
417 return (cp_slow0((vm_offset_t)base, sizeof(long), true,
418 suword_slow0, &word) != 0 ? -1 : 0);
422 suword32(volatile void *base, int32_t word)
425 return (suword(base, word));
428 struct casueword_arg0 {
435 casueword_slow0(vm_offset_t kva, void *arg)
437 struct casueword_arg0 *ca;
440 ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
445 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
448 struct casueword_arg0 ca;
453 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
454 casueword_slow0, &ca);
456 *oldvalp = ca.oldval;
463 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
465 struct casueword_arg0 ca;
470 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
471 casueword_slow0, &ca);
473 *oldvalp = ca.oldval;