2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
6 * This software was developed by Andrew Turner under sponsorship from
7 * the FreeBSD Foundation.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include "opt_kstack_pages.h"
34 #include "opt_sched.h"
36 #include <machine/asm.h>
38 __FBSDID("$FreeBSD$");
40 .macro clear_step_flag pcbflags, tmp
41 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
49 .macro set_step_flag pcbflags, tmp
50 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
59 * void cpu_throw(struct thread *old, struct thread *new)
62 /* Of old == NULL skip disabling stepping */
65 /* If we were single stepping, disable it */
67 ldr w5, [x4, #PCB_FLAGS]
68 clear_step_flag w5, x6
72 /* Backup the new thread pointer around a call to C code */
78 /* Store the new curthread */
79 str x1, [x18, #PC_CURTHREAD]
82 str x4, [x18, #PC_CURPCB]
85 * TODO: We may need to flush the cache here.
88 /* Switch to the new pmap */
89 ldr x5, [x4, #PCB_L1ADDR]
93 /* Invalidate the TLB */
99 /* If we are single stepping, enable it */
100 ldr w5, [x4, #PCB_FLAGS]
103 /* Restore the registers */
104 ldp x5, x6, [x4, #PCB_SP]
107 ldp x8, x9, [x4, #PCB_REGS + 8 * 8]
108 ldp x10, x11, [x4, #PCB_REGS + 10 * 8]
109 ldp x12, x13, [x4, #PCB_REGS + 12 * 8]
110 ldp x14, x15, [x4, #PCB_REGS + 14 * 8]
111 ldp x16, x17, [x4, #PCB_REGS + 16 * 8]
112 ldr x19, [x4, #PCB_REGS + 19 * 8]
113 ldp x20, x21, [x4, #PCB_REGS + 20 * 8]
114 ldp x22, x23, [x4, #PCB_REGS + 22 * 8]
115 ldp x24, x25, [x4, #PCB_REGS + 24 * 8]
116 ldp x26, x27, [x4, #PCB_REGS + 26 * 8]
117 ldp x28, x29, [x4, #PCB_REGS + 28 * 8]
118 ldr x30, [x4, #PCB_REGS + 30 * 8]
124 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
129 * x3 to x7, x16 and x17 are caller saved
132 /* Store the new curthread */
133 str x1, [x18, #PC_CURTHREAD]
134 /* And the new pcb */
135 ldr x4, [x1, #TD_PCB]
136 str x4, [x18, #PC_CURPCB]
139 * Save the old context.
141 ldr x4, [x0, #TD_PCB]
143 /* Store the callee-saved registers */
144 stp x8, x9, [x4, #PCB_REGS + 8 * 8]
145 stp x10, x11, [x4, #PCB_REGS + 10 * 8]
146 stp x12, x13, [x4, #PCB_REGS + 12 * 8]
147 stp x14, x15, [x4, #PCB_REGS + 14 * 8]
148 stp x16, x17, [x4, #PCB_REGS + 16 * 8]
149 stp x18, x19, [x4, #PCB_REGS + 18 * 8]
150 stp x20, x21, [x4, #PCB_REGS + 20 * 8]
151 stp x22, x23, [x4, #PCB_REGS + 22 * 8]
152 stp x24, x25, [x4, #PCB_REGS + 24 * 8]
153 stp x26, x27, [x4, #PCB_REGS + 26 * 8]
154 stp x28, x29, [x4, #PCB_REGS + 28 * 8]
155 str x30, [x4, #PCB_REGS + 30 * 8]
156 /* And the old stack pointer */
159 stp x5, x6, [x4, #PCB_SP]
161 /* If we were single stepping, disable it */
162 ldr w5, [x4, #PCB_FLAGS]
163 clear_step_flag w5, x6
169 /* Load the pcb address */
178 * Restore the saved context.
180 ldr x4, [x1, #TD_PCB]
183 * TODO: We may need to flush the cache here if switching
187 /* Switch to the new pmap */
188 ldr x5, [x4, #PCB_L1ADDR]
192 /* Invalidate the TLB */
199 * Release the old thread. This doesn't need to be a store-release
200 * as the above dsb instruction will provide release semantics.
202 str x2, [x0, #TD_LOCK]
203 #if defined(SCHED_ULE) && defined(SMP)
204 /* Spin if TD_LOCK points to a blocked_lock */
205 ldr x2, =_C_LABEL(blocked_lock)
207 ldar x3, [x1, #TD_LOCK]
212 /* If we are single stepping, enable it */
213 ldr w5, [x4, #PCB_FLAGS]
216 /* Restore the registers */
217 ldp x5, x6, [x4, #PCB_SP]
220 ldp x8, x9, [x4, #PCB_REGS + 8 * 8]
221 ldp x10, x11, [x4, #PCB_REGS + 10 * 8]
222 ldp x12, x13, [x4, #PCB_REGS + 12 * 8]
223 ldp x14, x15, [x4, #PCB_REGS + 14 * 8]
224 ldp x16, x17, [x4, #PCB_REGS + 16 * 8]
225 ldr x19, [x4, #PCB_REGS + 19 * 8]
226 ldp x20, x21, [x4, #PCB_REGS + 20 * 8]
227 ldp x22, x23, [x4, #PCB_REGS + 22 * 8]
228 ldp x24, x25, [x4, #PCB_REGS + 24 * 8]
229 ldp x26, x27, [x4, #PCB_REGS + 26 * 8]
230 ldp x28, x29, [x4, #PCB_REGS + 28 * 8]
231 ldr x30, [x4, #PCB_REGS + 30 * 8]
233 str xzr, [x4, #PCB_REGS + 18 * 8]
235 .Lcpu_switch_panic_str:
236 .asciz "cpu_switch: %p\0"
239 ENTRY(fork_trampoline)
243 mov fp, #0 /* Stack traceback stops here. */
244 bl _C_LABEL(fork_exit)
246 /* Restore sp and lr */
251 /* Restore the registers other than x0 and x1 */
252 ldp x2, x3, [sp, #TF_X + 2 * 8]
253 ldp x4, x5, [sp, #TF_X + 4 * 8]
254 ldp x6, x7, [sp, #TF_X + 6 * 8]
255 ldp x8, x9, [sp, #TF_X + 8 * 8]
256 ldp x10, x11, [sp, #TF_X + 10 * 8]
257 ldp x12, x13, [sp, #TF_X + 12 * 8]
258 ldp x14, x15, [sp, #TF_X + 14 * 8]
259 ldp x16, x17, [sp, #TF_X + 16 * 8]
260 ldr x19, [sp, #TF_X + 19 * 8]
261 ldp x20, x21, [sp, #TF_X + 20 * 8]
262 ldp x22, x23, [sp, #TF_X + 22 * 8]
263 ldp x24, x25, [sp, #TF_X + 24 * 8]
264 ldp x26, x27, [sp, #TF_X + 26 * 8]
265 ldp x28, x29, [sp, #TF_X + 28 * 8]
266 /* Skip x30 as it was restored above as lr */
269 * Disable interrupts to avoid
270 * overwriting spsr_el1 by an IRQ exception.
274 /* Restore elr and spsr */
275 ldp x0, x1, [sp, #16]
279 /* Finally x0 and x1 */
280 ldp x0, x1, [sp, #TF_X + 0 * 8]
281 ldr x18, [sp, #TF_X + 18 * 8]
284 * No need for interrupts reenabling since PSR
285 * will be set to the desired value anyway.
292 /* Store the callee-saved registers */
293 stp x8, x9, [x0, #PCB_REGS + 8 * 8]
294 stp x10, x11, [x0, #PCB_REGS + 10 * 8]
295 stp x12, x13, [x0, #PCB_REGS + 12 * 8]
296 stp x14, x15, [x0, #PCB_REGS + 14 * 8]
297 stp x16, x17, [x0, #PCB_REGS + 16 * 8]
298 stp x18, x19, [x0, #PCB_REGS + 18 * 8]
299 stp x20, x21, [x0, #PCB_REGS + 20 * 8]
300 stp x22, x23, [x0, #PCB_REGS + 22 * 8]
301 stp x24, x25, [x0, #PCB_REGS + 24 * 8]
302 stp x26, x27, [x0, #PCB_REGS + 26 * 8]
303 stp x28, x29, [x0, #PCB_REGS + 28 * 8]
304 str x30, [x0, #PCB_REGS + 30 * 8]
305 /* And the old stack pointer */
308 stp x5, x6, [x0, #PCB_SP]
310 /* Store the VFP registers */
313 mov x1, x0 /* move pcb to the correct register */
314 mov x0, xzr /* td = NULL */