2 * Copyright (c) 2015 The FreeBSD Foundation
5 * This software was developed by Andrew Turner under
6 * sponsorship from the FreeBSD Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <machine/asm.h>
32 __FBSDID("$FreeBSD$");
34 #include <sys/errno.h>
36 #include <machine/vmparam.h>
41 * Fault handler for the copy{in,out} functions below.
44 SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */
45 EXIT_USER_ACCESS_CHECK(w0, x1)
52 * Copies from a kernel to user address
54 * int copyout(const void *kaddr, void *udaddr, size_t len)
59 b.cs copyio_fault_nopcb
60 ldr x4, =VM_MAXUSER_ADDRESS
62 b.hi copyio_fault_nopcb
66 1: mov x0, xzr /* return 0 */
72 * Copies from a user to kernel address
74 * int copyin(const void *uaddr, void *kdaddr, size_t len)
79 b.cs copyio_fault_nopcb
80 ldr x4, =VM_MAXUSER_ADDRESS
82 b.hi copyio_fault_nopcb
86 1: mov x0, xzr /* return 0 */
92 * Copies a string from a user to kernel address
94 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
97 mov x5, xzr /* count = 0 */
98 mov w4, #1 /* If zero return faulure */
99 cbz x2, 3f /* If len == 0 then skip loop */
101 adr x6, copyio_fault /* Get the handler address */
102 SET_FAULT_HANDLER(x6, x7) /* Set the handler */
103 ENTER_USER_ACCESS(w6, x7)
105 ldr x7, =VM_MAXUSER_ADDRESS
108 ldtrb w4, [x0] /* Load from uaddr */
109 add x0, x0, #1 /* Next char */
110 strb w4, [x1], #1 /* Store in kaddr */
111 add x5, x5, #1 /* count++ */
112 cbz w4, 2f /* Break when NUL-terminated */
113 sub x2, x2, #1 /* len-- */
116 2: EXIT_USER_ACCESS(w6)
117 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
120 3: cbz x3, 4f /* Check if done != NULL */
121 str x5, [x3] /* done = count */
123 4: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */
124 cmp w4, #0 /* Check if we saved the NUL-terminator */
125 csel w0, wzr, w1, eq /* If so return success, else failure */
135 * lr - the return address, so jump here instead of calling
137 * This function is optimized to minimize concurrent memory accesses. In
138 * present form it is suited for cores with a single memory prefetching
141 * Consider using separate functions for each ARM64 core. Adding memory
142 * access interleaving might increase a total throughput on A57 or A72.
147 .type copycommon,@function
150 adr x6, copyio_fault /* Get the handler address */
151 SET_FAULT_HANDLER(x6, x7) /* Set the handler */
152 ENTER_USER_ACCESS(w6, x7)
154 /* Check alignment */
159 /* Unaligned is byte by byte copy */
176 ldp x3, x4, [x0], #0x10
177 ldp x5, x6, [x0], #0x10
178 ldp x7, x8, [x0], #0x10
179 ldp x9, x10, [x0], #0x10
180 stp x3, x4, [x1], #0x10
181 stp x5, x6, [x1], #0x10
182 stp x7, x8, [x1], #0x10
183 stp x9, x10, [x1], #0x10
194 ldp x3, x4, [x0], #0x10
195 stp x3, x4, [x1], #0x10
199 /* Less than 16 bytes to copy */
201 tbz x2, #0x03, last_word
206 tbz x2, #0x02, last_hword
211 tbz x2, #0x01, last_byte
216 tbz x2, #0x00, ending
221 EXIT_USER_ACCESS_CHECK(w6, x7)
222 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
224 mov x0, xzr /* return 0 */
226 .size copycommon, . - copycommon