1 /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */
4 * Copyright 2003 Wasabi Systems, Inc.
7 * Written by Steve C. Woodford for Wasabi Systems, Inc.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
38 * Copyright (c) 1994-1998 Mark Brinicombe.
39 * Copyright (c) 1994 Brini.
40 * All rights reserved.
42 * This code is derived from software written for Brini by Mark Brinicombe
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. All advertising materials mentioning features or use of this software
53 * must display the following acknowledgement:
54 * This product includes software developed by Brini.
55 * 4. The name of the company nor the name of the author may be used to
56 * endorse or promote products derived from this software without specific
57 * prior written permission.
59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
71 * RiscBSD kernel project
75 * cpu switching functions
82 #include "opt_sched.h"
84 #include <machine/asm.h>
85 #include <machine/asmacros.h>
86 #include <machine/armreg.h>
87 __FBSDID("$FreeBSD$");
89 #define DOMAIN_CLIENT 0x01
91 #if defined(_ARM_ARCH_6) && defined(SMP)
92 #define GET_PCPU(tmp, tmp2) \
93 mrc p15, 0, tmp, c0, c0, 5; \
95 ldr tmp2, .Lcurpcpu+4; \
97 ldr tmp2, .Lcurpcpu; \
101 #define GET_PCPU(tmp, tmp2) \
106 .word _C_LABEL(__pcpu)
109 .word _C_LABEL(cpufuncs)
111 .word _C_LABEL(blocked_lock)
125 * vfp_discard will clear pcpu->pc_vfpcthread, and modify
126 * and modify the control as needed.
128 ldr r4, [r7, #(PC_VFPCTHREAD)] /* this thread using vfp? */
131 bl _C_LABEL(vfp_discard) /* yes, shut down vfp */
135 ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */
137 /* Switch to lwp0 context */
140 #if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) && !defined(CPU_KRAIT)
142 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL]
144 ldr r0, [r7, #(PCB_PL1VEC)]
145 ldr r1, [r7, #(PCB_DACR)]
147 * r0 = Pointer to L1 slot for vector_page (or NULL)
156 * Ensure the vector table is accessible by fixing up lwp0's L1
158 cmp r0, #0 /* No need to fixup vector table? */
159 ldrne r3, [r0] /* But if yes, fetch current value */
160 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */
161 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */
162 cmpne r3, r2 /* Stuffing the same value? */
163 strne r2, [r0] /* Store if not. */
165 #ifdef PMAP_INCLUDE_PTE_SYNC
167 * Need to sync the cache to make sure that last store is
168 * visible to the MMU.
172 ldrne pc, [r9, #CF_DCACHE_WB_RANGE]
173 #endif /* PMAP_INCLUDE_PTE_SYNC */
176 * Note: We don't do the same optimisation as cpu_switch() with
177 * respect to avoiding flushing the TLB if we're switching to
178 * the same L1 since this process' VM space may be about to go
179 * away, so we don't want *any* turds left in the TLB.
182 /* Switch the memory to the new process */
183 ldr r0, [r7, #(PCB_PAGEDIR)]
185 ldr pc, [r9, #CF_CONTEXT_SWITCH]
187 /* Restore all the save registers */
192 ldr r8, [r7, #(PCB_R8)]
193 ldr r9, [r7, #(PCB_R9)]
194 ldr r10, [r7, #(PCB_R10)]
195 ldr r11, [r7, #(PCB_R11)]
196 ldr r12, [r7, #(PCB_R12)]
197 ldr r13, [r7, #(PCB_SP)]
201 /* Hook in a new pcb */
202 str r7, [r6, #PC_CURPCB]
203 /* We have a new curthread now so make a note it */
204 add r6, r6, #PC_CURTHREAD
206 #ifndef ARM_TP_ADDRESS
207 mcr p15, 0, r5, c13, c0, 4
210 ldr r6, [r5, #(TD_MD + MD_TP)]
211 #ifdef ARM_TP_ADDRESS
212 ldr r4, =ARM_TP_ADDRESS
214 ldr r6, [r5, #(TD_MD + MD_RAS_START)]
215 str r6, [r4, #4] /* ARM_RAS_START */
216 ldr r6, [r5, #(TD_MD + MD_RAS_END)]
217 str r6, [r4, #8] /* ARM_RAS_END */
219 mcr p15, 0, r6, c13, c0, 3
223 ldmfd sp!, {r4-r7, pc}
227 stmfd sp!, {r4-r7, lr}
234 mov r6, r2 /* Save the mutex */
237 /* rem: r0 = old lwp */
238 /* rem: interrupts are disabled */
240 /* Process is now on a processor. */
241 /* We have a new curthread now so make a note it */
243 str r1, [r7, #PC_CURTHREAD]
244 #ifndef ARM_TP_ADDRESS
245 mcr p15, 0, r1, c13, c0, 4
248 /* Hook in a new pcb */
249 ldr r2, [r1, #TD_PCB]
250 str r2, [r7, #PC_CURPCB]
252 /* rem: r1 = new process */
253 /* rem: interrupts are enabled */
255 /* Stage two : Save old context */
257 /* Get the user structure for the old thread. */
258 ldr r2, [r0, #(TD_PCB)]
259 mov r4, r0 /* Save the old thread. */
261 /* Save all the registers in the old thread's pcb */
263 add r7, r2, #(PCB_R8)
266 strd r8, [r2, #(PCB_R8)]
267 strd r10, [r2, #(PCB_R10)]
268 strd r12, [r2, #(PCB_R12)]
270 str pc, [r2, #(PCB_PC)]
273 * NOTE: We can now use r8-r13 until it is time to restore
274 * them for the new process.
276 #ifdef ARM_TP_ADDRESS
277 /* Store the old tp */
278 ldr r3, =ARM_TP_ADDRESS
280 str r9, [r0, #(TD_MD + MD_TP)]
282 str r9, [r0, #(TD_MD + MD_RAS_START)]
284 str r9, [r0, #(TD_MD + MD_RAS_END)]
287 ldr r9, [r1, #(TD_MD + MD_TP)]
289 ldr r9, [r1, #(TD_MD + MD_RAS_START)]
291 ldr r9, [r1, #(TD_MD + MD_RAS_END)]
294 /* Store the old tp */
295 mrc p15, 0, r9, c13, c0, 3
296 str r9, [r0, #(TD_MD + MD_TP)]
299 ldr r9, [r1, #(TD_MD + MD_TP)]
300 mcr p15, 0, r9, c13, c0, 3
303 /* Get the user structure for the new process in r9 */
304 ldr r9, [r1, #(TD_PCB)]
308 * We can do that, since
309 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE
311 orr r8, r3, #(PSR_UND32_MODE)
314 str sp, [r2, #(PCB_UND_SP)]
316 msr cpsr_c, r3 /* Restore the old mode */
317 /* rem: r2 = old PCB */
318 /* rem: r9 = new PCB */
319 /* rem: interrupts are enabled */
323 * vfp_store will clear pcpu->pc_vfpcthread, save
324 * registers and state, and modify the control as needed.
325 * a future exception will bounce the backup settings in the fp unit.
326 * XXX vfp_store can't change r4
329 ldr r8, [r7, #(PC_VFPCTHREAD)]
330 cmp r4, r8 /* old thread used vfp? */
331 bne 1f /* no, don't save */
332 cmp r1, r4 /* same thread ? */
333 beq 1f /* yes, skip vfp store */
335 ldr r8, [r7, #(PC_CPU)] /* last used on this cpu? */
336 ldr r3, [r2, #(PCB_VFPCPU)]
337 cmp r8, r3 /* last cpu to use these registers? */
338 bne 1f /* no. these values are stale */
340 add r0, r2, #(PCB_VFPSTATE)
341 bl _C_LABEL(vfp_store)
347 /* Third phase : restore saved context */
349 /* rem: r2 = old PCB */
350 /* rem: r9 = new PCB */
351 /* rem: interrupts are enabled */
353 ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */
354 mov r2, #DOMAIN_CLIENT
355 cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */
356 beq .Lcs_context_switched /* Yup. Don't flush cache */
357 mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */
359 * Get the new L1 table pointer into r11. If we're switching to
360 * an LWP with the same address space as the outgoing one, we can
361 * skip the cache purge and the TTB load.
363 * To avoid data dep stalls that would happen anyway, we try
364 * and get some useful work done in the mean time.
366 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */
367 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */
370 teq r10, r11 /* Same L1? */
371 cmpeq r0, r5 /* Same DACR? */
372 beq .Lcs_context_switched /* yes! */
374 #if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) && !defined(CPU_KRAIT)
376 * Definately need to flush the cache.
381 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL]
383 .Lcs_cache_purge_skipped:
385 /* rem: r9 = new PCB */
386 /* rem: r10 = old L1 */
387 /* rem: r11 = new L1 */
390 ldr r7, [r9, #(PCB_PL1VEC)]
393 * Ensure the vector table is accessible by fixing up the L1
395 cmp r7, #0 /* No need to fixup vector table? */
396 ldrne r2, [r7] /* But if yes, fetch current value */
397 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */
398 mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */
399 cmpne r2, r0 /* Stuffing the same value? */
400 #ifndef PMAP_INCLUDE_PTE_SYNC
401 strne r0, [r7] /* Nope, update it */
404 str r0, [r7] /* Otherwise, update it */
407 * Need to sync the cache to make sure that last store is
408 * visible to the MMU.
414 ldr pc, [r2, #CF_DCACHE_WB_RANGE]
417 #endif /* PMAP_INCLUDE_PTE_SYNC */
419 cmp r10, r11 /* Switching to the same L1? */
421 beq .Lcs_same_l1 /* Yup. */
423 * Do a full context switch, including full TLB flush.
427 ldr pc, [r10, #CF_CONTEXT_SWITCH]
429 b .Lcs_context_switched
432 * We're switching to a different process in the same L1.
433 * In this situation, we only need to flush the TLB for the
434 * vector_page mapping, and even then only if r7 is non-NULL.
438 movne r0, #0 /* We *know* vector_page's VA is 0x0 */
440 ldrne pc, [r10, #CF_TLB_FLUSHID_SE]
442 * We can do that, since
443 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE
446 .Lcs_context_switched:
448 /* Release the old thread */
449 str r6, [r4, #TD_LOCK]
450 #if defined(SCHED_ULE) && defined(SMP)
451 ldr r6, .Lblocked_lock
452 GET_CURTHREAD_PTR(r3)
454 ldr r4, [r3, #TD_LOCK]
459 /* XXXSCW: Safe to re-enable FIQs here */
461 /* rem: r9 = new PCB */
465 * We can do that, since
466 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE
468 orr r2, r3, #(PSR_UND32_MODE)
471 ldr sp, [r9, #(PCB_UND_SP)]
473 msr cpsr_c, r3 /* Restore the old mode */
474 /* Restore all the save registers */
478 sub r7, r7, #PCB_R8 /* restore PCB pointer */
481 ldr r8, [r7, #(PCB_R8)]
482 ldr r9, [r7, #(PCB_R9)]
483 ldr r10, [r7, #(PCB_R10)]
484 ldr r11, [r7, #(PCB_R11)]
485 ldr r12, [r7, #(PCB_R12)]
486 ldr r13, [r7, #(PCB_SP)]
489 /* rem: r5 = new lwp's proc */
491 /* rem: r7 = new PCB */
496 * Pull the registers that got pushed when either savectx() or
497 * cpu_switch() was called and return.
500 ldmfd sp!, {r4-r7, pc}
503 adr r0, .Lswitch_panic_str
509 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n"
514 stmfd sp!, {r4-r7, lr}
519 /* Store all the registers in the process's pcb */
520 add r2, r0, #(PCB_R8)
524 * vfp_store will clear pcpu->pc_vfpcthread, save
525 * registers and state, and modify the control as needed.
526 * a future exception will bounce the backup settings in the fp unit.
529 ldr r4, [r7, #(PC_VFPCTHREAD)] /* vfp thread */
530 ldr r2, [r7, #(PC_CURTHREAD)] /* current thread */
534 ldr r2, [r7, #(PC_CPU)] /* last used on this cpu? */
535 ldr r3, [r0, #(PCB_VFPCPU)]
537 bne 1f /* no. these values are stale */
539 add r0, r0, #(PCB_VFPSTATE)
540 bl _C_LABEL(vfp_store)
544 ldmfd sp!, {r4-r7, pc}
547 ENTRY(fork_trampoline)
548 STOP_UNWINDING /* Can't unwind beyond the thread enty point */
553 bl _C_LABEL(fork_exit)
556 orr r0, r0, #(I32_bit|F32_bit)
561 movs pc, lr /* Exit */