2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014-2015 The FreeBSD Foundation
6 * Portions of this software were developed by Andrew Turner
7 * under sponsorship from the FreeBSD Foundation
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <machine/asm.h>
33 __FBSDID("$FreeBSD$");
35 #include <machine/setjmp.h>
36 #include <machine/param.h>
37 #include <machine/vmparam.h>
42 * One of the fu* or su* functions failed, return -1.
45 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */
46 EXIT_USER_ACCESS_CHECK(w0, x1)
53 * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
56 ldr x4, =(VM_MAXUSER_ADDRESS-3)
59 adr x6, fsu_fault /* Load the fault handler */
60 SET_FAULT_HANDLER(x6, x4) /* And set it */
61 ENTER_USER_ACCESS(w6, x4)
62 1: ldxr w4, [x0] /* Load-exclusive the data */
63 cmp w4, w1 /* Compare */
64 b.ne 2f /* Not equal, exit */
65 stxr w5, w3, [x0] /* Store the new data */
66 cbnz w5, 1b /* Retry on failure */
68 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
69 str w4, [x2] /* Store the read data */
70 mov x0, #0 /* Success */
75 * int casueword(volatile u_long *, u_long, u_long *, u_long)
78 ldr x4, =(VM_MAXUSER_ADDRESS-7)
81 adr x6, fsu_fault /* Load the fault handler */
82 SET_FAULT_HANDLER(x6, x4) /* And set it */
83 ENTER_USER_ACCESS(w6, x4)
84 1: ldxr x4, [x0] /* Load-exclusive the data */
85 cmp x4, x1 /* Compare */
86 b.ne 2f /* Not equal, exit */
87 stxr w5, x3, [x0] /* Store the new data */
88 cbnz w5, 1b /* Retry on failure */
90 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
91 str x4, [x2] /* Store the read data */
92 mov x0, #0 /* Success */
97 * int fubyte(volatile const void *)
100 ldr x1, =VM_MAXUSER_ADDRESS
103 adr x6, fsu_fault /* Load the fault handler */
104 SET_FAULT_HANDLER(x6, x1) /* And set it */
105 ldtrb w0, [x0] /* Try loading the data */
106 SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */
111 * int fuword(volatile const void *)
114 ldr x1, =(VM_MAXUSER_ADDRESS-1)
117 adr x6, fsu_fault /* Load the fault handler */
118 SET_FAULT_HANDLER(x6, x1) /* And set it */
119 ldtrh w0, [x0] /* Try loading the data */
120 SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */
125 * int32_t fueword32(volatile const void *, int32_t *)
128 ldr x2, =(VM_MAXUSER_ADDRESS-3)
131 adr x6, fsu_fault /* Load the fault handler */
132 SET_FAULT_HANDLER(x6, x2) /* And set it */
133 ldtr w0, [x0] /* Try loading the data */
134 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
135 str w0, [x1] /* Save the data in kernel space */
136 mov w0, #0 /* Success */
141 * long fueword(volatile const void *, int64_t *)
142 * int64_t fueword64(volatile const void *, int64_t *)
146 ldr x2, =(VM_MAXUSER_ADDRESS-7)
149 adr x6, fsu_fault /* Load the fault handler */
150 SET_FAULT_HANDLER(x6, x2) /* And set it */
151 ldtr x0, [x0] /* Try loading the data */
152 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
153 str x0, [x1] /* Save the data in kernel space */
154 mov x0, #0 /* Success */
160 * int subyte(volatile void *, int)
163 ldr x2, =VM_MAXUSER_ADDRESS
166 adr x6, fsu_fault /* Load the fault handler */
167 SET_FAULT_HANDLER(x6, x2) /* And set it */
168 sttrb w1, [x0] /* Try storing the data */
169 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
170 mov x0, #0 /* Success */
175 * int suword16(volatile void *, int)
178 ldr x2, =(VM_MAXUSER_ADDRESS-1)
181 adr x6, fsu_fault /* Load the fault handler */
182 SET_FAULT_HANDLER(x6, x2) /* And set it */
183 sttrh w1, [x0] /* Try storing the data */
184 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
185 mov x0, #0 /* Success */
190 * int suword32(volatile void *, int)
193 ldr x2, =(VM_MAXUSER_ADDRESS-3)
196 adr x6, fsu_fault /* Load the fault handler */
197 SET_FAULT_HANDLER(x6, x2) /* And set it */
198 sttr w1, [x0] /* Try storing the data */
199 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
200 mov x0, #0 /* Success */
205 * int suword(volatile void *, long)
209 ldr x2, =(VM_MAXUSER_ADDRESS-7)
212 adr x6, fsu_fault /* Load the fault handler */
213 SET_FAULT_HANDLER(x6, x2) /* And set it */
214 sttr x1, [x0] /* Try storing the data */
215 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
216 mov x0, #0 /* Success */
222 * fuswintr and suswintr are just like fusword and susword except that if
223 * the page is not in memory or would cause a trap, then we return an error.
224 * The important thing is to prevent sleep() and switch().
228 * Special handler so the trap code knows not to sleep.
230 ENTRY(fsu_intr_fault)
231 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */
232 EXIT_USER_ACCESS_CHECK(w0, x1)
238 * int fuswintr(void *)
241 ldr x1, =(VM_MAXUSER_ADDRESS-3)
244 adr x6, fsu_intr_fault /* Load the fault handler */
245 SET_FAULT_HANDLER(x6, x1) /* And set it */
246 ldtr w0, [x0] /* Try loading the data */
247 SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */
252 * int suswintr(void *base, int word)
255 ldr x2, =(VM_MAXUSER_ADDRESS-3)
258 adr x6, fsu_intr_fault /* Load the fault handler */
259 SET_FAULT_HANDLER(x6, x2) /* And set it */
260 sttr w1, [x0] /* Try storing the data */
261 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */
262 mov x0, #0 /* Success */
267 /* Store the stack pointer */
271 /* Store the general purpose registers and lr */
272 stp x19, x20, [x0], #16
273 stp x21, x22, [x0], #16
274 stp x23, x24, [x0], #16
275 stp x25, x26, [x0], #16
276 stp x27, x28, [x0], #16
277 stp x29, lr, [x0], #16
285 /* Restore the stack pointer */
289 /* Restore the general purpose registers and lr */
290 ldp x19, x20, [x0], #16
291 ldp x21, x22, [x0], #16
292 ldp x23, x24, [x0], #16
293 ldp x25, x26, [x0], #16
294 ldp x27, x28, [x0], #16
295 ldp x29, lr, [x0], #16
297 /* Load the return value */
303 * pagezero, simple implementation
305 ENTRY(pagezero_simple)
306 add x1, x0, #PAGE_SIZE
309 stp xzr, xzr, [x0], #0x10
310 stp xzr, xzr, [x0], #0x10
311 stp xzr, xzr, [x0], #0x10
312 stp xzr, xzr, [x0], #0x10
320 * pagezero, cache assisted
322 ENTRY(pagezero_cache)
323 add x1, x0, #PAGE_SIZE
325 ldr x2, =dczva_line_size