2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014-2015 The FreeBSD Foundation
6 * Portions of this software were developed by Andrew Turner
7 * under sponsorship from the FreeBSD Foundation
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <machine/asm.h>
33 #include <machine/setjmp.h>
34 #include <machine/param.h>
35 #include <machine/vmparam.h>
39 .macro check_user_access user_arg, limit, bad_addr_func
46 * One of the fu* or su* functions failed, return -1.
49 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */
50 EXIT_USER_ACCESS_CHECK(w0, x1)
57 * int swapueword8_llsc(volatile uint8_t *, uint8_t *)
59 ENTRY(swapueword8_llsc)
60 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
61 adr x6, fsu_fault /* Load the fault handler */
62 SET_FAULT_HANDLER(x6, x4) /* And set it */
63 ENTER_USER_ACCESS(w6, x4)
71 strb w2, [x1] /* Stash old value in *val */
73 1: EXIT_USER_ACCESS(w6)
74 SET_FAULT_HANDLER(xzr, x6)
80 * int swapueword8_lse(volatile uint8_t *, uint8_t *)
82 ENTRY(swapueword8_lse)
83 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
84 adr x6, fsu_fault /* Load the fault handler */
85 SET_FAULT_HANDLER(x6, x4) /* And set it */
86 ENTER_USER_ACCESS(w6, x4)
94 strb w2, [x1] /* Stash old value in *val */
97 SET_FAULT_HANDLER(xzr, x6)
103 * int swapueword32_llsc(volatile uint32_t *, uint32_t *)
105 ENTRY(swapueword32_llsc)
106 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
107 adr x6, fsu_fault /* Load the fault handler */
108 SET_FAULT_HANDLER(x6, x4) /* And set it */
109 ENTER_USER_ACCESS(w6, x4)
113 ldxr w2, [x0] /* Stash the old value in w2 */
114 stxr w3, w7, [x0] /* Store new value */
117 str w2, [x1] /* Stash old value in *val */
119 1: EXIT_USER_ACCESS(w6)
120 SET_FAULT_HANDLER(xzr, x6)
123 END(swapueword32_llsc)
126 * int swapueword32_lse(volatile uint32_t *, uint32_t *)
128 ENTRY(swapueword32_lse)
129 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
130 adr x6, fsu_fault /* Load the fault handler */
131 SET_FAULT_HANDLER(x6, x4) /* And set it */
132 ENTER_USER_ACCESS(w6, x4)
138 .arch_extension nolse
140 str w2, [x1] /* Stash old value in *val */
143 SET_FAULT_HANDLER(xzr, x6)
146 END(swapueword32_llsc)
149 * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
151 ENTRY(casueword32_llsc)
152 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
153 adr x6, fsu_fault /* Load the fault handler */
155 SET_FAULT_HANDLER(x6, x4) /* And set it */
156 ENTER_USER_ACCESS(w6, x4)
157 ldxr w4, [x0] /* Load-exclusive the data */
158 cmp w4, w1 /* Compare */
159 b.ne 1f /* Not equal, exit */
160 stxr w5, w3, [x0] /* Store the new data */
161 1: EXIT_USER_ACCESS(w6)
162 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
163 str w4, [x2] /* Store the read data */
164 mov w0, w5 /* Result same as store status */
166 END(casueword32_llsc)
169 * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
171 ENTRY(casueword32_lse)
172 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
173 adr x6, fsu_fault /* Load the fault handler */
174 SET_FAULT_HANDLER(x6, x4) /* And set it */
175 ENTER_USER_ACCESS(w6, x4)
176 mov w7, w1 /* Back up the compare value */
178 cas w1, w3, [x0] /* Compare and Swap */
179 .arch_extension nolse
180 cmp w1, w7 /* Check if successful */
181 cset w0, ne /* Return 0 on success, 1 on failure */
183 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
184 str w1, [x2] /* Store the read data */
189 * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long)
191 ENTRY(casueword_llsc)
192 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb
193 adr x6, fsu_fault /* Load the fault handler */
195 SET_FAULT_HANDLER(x6, x4) /* And set it */
196 ENTER_USER_ACCESS(w6, x4)
197 ldxr x4, [x0] /* Load-exclusive the data */
198 cmp x4, x1 /* Compare */
199 b.ne 1f /* Not equal, exit */
200 stxr w5, x3, [x0] /* Store the new data */
201 1: EXIT_USER_ACCESS(w6)
202 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
203 str x4, [x2] /* Store the read data */
204 mov w0, w5 /* Result same as store status */
209 * int casueword_lse(volatile u_long *, u_long, u_long *, u_long)
212 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
213 adr x6, fsu_fault /* Load the fault handler */
214 SET_FAULT_HANDLER(x6, x4) /* And set it */
215 ENTER_USER_ACCESS(w6, x4)
216 mov x7, x1 /* Back up the compare value */
218 cas x1, x3, [x0] /* Compare and Swap */
219 .arch_extension nolse
220 cmp x1, x7 /* Check if successful */
221 cset w0, ne /* Return 0 on success, 1 on failure */
223 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
224 str x1, [x2] /* Store the read data */
228 .macro fsudata insn, ret_reg, user_arg
229 adr x7, fsu_fault /* Load the fault handler */
230 SET_FAULT_HANDLER(x7, x6) /* And set it */
231 \insn \ret_reg, [x\user_arg] /* Try accessing the data */
232 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
236 * int fubyte(volatile const void *)
239 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb
245 * int fuword(volatile const void *)
248 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb
254 * int32_t fueword32(volatile const void *, int32_t *)
257 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
259 str w0, [x1] /* Save the data in kernel space */
260 mov w0, #0 /* Success */
265 * long fueword(volatile const void *, int64_t *)
266 * int64_t fueword64(volatile const void *, int64_t *)
270 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb
272 str x0, [x1] /* Save the data in kernel space */
273 mov x0, #0 /* Success */
279 * int subyte(volatile void *, int)
282 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb
284 mov x0, #0 /* Success */
289 * int suword16(volatile void *, int)
292 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb
294 mov x0, #0 /* Success */
299 * int suword32(volatile void *, int)
302 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
304 mov x0, #0 /* Success */
309 * int suword(volatile void *, long)
313 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb
315 mov x0, #0 /* Success */
321 /* Store the stack pointer */
325 /* Store the general purpose registers and lr */
326 stp x19, x20, [x0], #16
327 stp x21, x22, [x0], #16
328 stp x23, x24, [x0], #16
329 stp x25, x26, [x0], #16
330 stp x27, x28, [x0], #16
331 stp x29, lr, [x0], #16
339 /* Restore the stack pointer */
343 /* Restore the general purpose registers and lr */
344 ldp x19, x20, [x0], #16
345 ldp x21, x22, [x0], #16
346 ldp x23, x24, [x0], #16
347 ldp x25, x26, [x0], #16
348 ldp x27, x28, [x0], #16
349 ldp x29, lr, [x0], #16
351 /* Load the return value */
357 * pagezero, simple implementation
359 ENTRY(pagezero_simple)
360 add x1, x0, #PAGE_SIZE
363 stp xzr, xzr, [x0], #0x10
364 stp xzr, xzr, [x0], #0x10
365 stp xzr, xzr, [x0], #0x10
366 stp xzr, xzr, [x0], #0x10
374 * pagezero, cache assisted
376 ENTRY(pagezero_cache)
377 add x1, x0, #PAGE_SIZE
379 adrp x2, dczva_line_size
380 ldr x2, [x2, :lo12:dczva_line_size]