2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include "opt_sched.h"
38 #include <machine/param.h>
39 #include <machine/asm.h>
40 #include <machine/riscvreg.h>
41 #include <machine/pte.h>
43 __FBSDID("$FreeBSD$");
46 .macro __fpe_state_save p
48 * Enable FPE usage in supervisor mode,
49 * so we can access registers.
51 li t0, SSTATUS_FS_INITIAL
57 fsd f0, (PCB_X + 0 * 16)(\p)
58 fsd f1, (PCB_X + 1 * 16)(\p)
59 fsd f2, (PCB_X + 2 * 16)(\p)
60 fsd f3, (PCB_X + 3 * 16)(\p)
61 fsd f4, (PCB_X + 4 * 16)(\p)
62 fsd f5, (PCB_X + 5 * 16)(\p)
63 fsd f6, (PCB_X + 6 * 16)(\p)
64 fsd f7, (PCB_X + 7 * 16)(\p)
65 fsd f8, (PCB_X + 8 * 16)(\p)
66 fsd f9, (PCB_X + 9 * 16)(\p)
67 fsd f10, (PCB_X + 10 * 16)(\p)
68 fsd f11, (PCB_X + 11 * 16)(\p)
69 fsd f12, (PCB_X + 12 * 16)(\p)
70 fsd f13, (PCB_X + 13 * 16)(\p)
71 fsd f14, (PCB_X + 14 * 16)(\p)
72 fsd f15, (PCB_X + 15 * 16)(\p)
73 fsd f16, (PCB_X + 16 * 16)(\p)
74 fsd f17, (PCB_X + 17 * 16)(\p)
75 fsd f18, (PCB_X + 18 * 16)(\p)
76 fsd f19, (PCB_X + 19 * 16)(\p)
77 fsd f20, (PCB_X + 20 * 16)(\p)
78 fsd f21, (PCB_X + 21 * 16)(\p)
79 fsd f22, (PCB_X + 22 * 16)(\p)
80 fsd f23, (PCB_X + 23 * 16)(\p)
81 fsd f24, (PCB_X + 24 * 16)(\p)
82 fsd f25, (PCB_X + 25 * 16)(\p)
83 fsd f26, (PCB_X + 26 * 16)(\p)
84 fsd f27, (PCB_X + 27 * 16)(\p)
85 fsd f28, (PCB_X + 28 * 16)(\p)
86 fsd f29, (PCB_X + 29 * 16)(\p)
87 fsd f30, (PCB_X + 30 * 16)(\p)
88 fsd f31, (PCB_X + 31 * 16)(\p)
90 /* Disable FPE usage in supervisor mode. */
91 li t0, SSTATUS_FS_MASK
95 .macro __fpe_state_load p
97 * Enable FPE usage in supervisor mode,
98 * so we can access registers.
100 li t0, SSTATUS_FS_INITIAL
103 /* Restore registers */
104 ld t0, (PCB_FCSR)(\p)
106 fld f0, (PCB_X + 0 * 16)(\p)
107 fld f1, (PCB_X + 1 * 16)(\p)
108 fld f2, (PCB_X + 2 * 16)(\p)
109 fld f3, (PCB_X + 3 * 16)(\p)
110 fld f4, (PCB_X + 4 * 16)(\p)
111 fld f5, (PCB_X + 5 * 16)(\p)
112 fld f6, (PCB_X + 6 * 16)(\p)
113 fld f7, (PCB_X + 7 * 16)(\p)
114 fld f8, (PCB_X + 8 * 16)(\p)
115 fld f9, (PCB_X + 9 * 16)(\p)
116 fld f10, (PCB_X + 10 * 16)(\p)
117 fld f11, (PCB_X + 11 * 16)(\p)
118 fld f12, (PCB_X + 12 * 16)(\p)
119 fld f13, (PCB_X + 13 * 16)(\p)
120 fld f14, (PCB_X + 14 * 16)(\p)
121 fld f15, (PCB_X + 15 * 16)(\p)
122 fld f16, (PCB_X + 16 * 16)(\p)
123 fld f17, (PCB_X + 17 * 16)(\p)
124 fld f18, (PCB_X + 18 * 16)(\p)
125 fld f19, (PCB_X + 19 * 16)(\p)
126 fld f20, (PCB_X + 20 * 16)(\p)
127 fld f21, (PCB_X + 21 * 16)(\p)
128 fld f22, (PCB_X + 22 * 16)(\p)
129 fld f23, (PCB_X + 23 * 16)(\p)
130 fld f24, (PCB_X + 24 * 16)(\p)
131 fld f25, (PCB_X + 25 * 16)(\p)
132 fld f26, (PCB_X + 26 * 16)(\p)
133 fld f27, (PCB_X + 27 * 16)(\p)
134 fld f28, (PCB_X + 28 * 16)(\p)
135 fld f29, (PCB_X + 29 * 16)(\p)
136 fld f30, (PCB_X + 30 * 16)(\p)
137 fld f31, (PCB_X + 31 * 16)(\p)
139 /* Disable FPE usage in supervisor mode. */
140 li t0, SSTATUS_FS_MASK
146 * fpe_state_save(struct thread *td)
148 ENTRY(fpe_state_save)
149 /* Get pointer to PCB */
158 * fpe_state_clear(void)
160 ENTRY(fpe_state_clear)
162 * Enable FPE usage in supervisor mode,
163 * so we can access registers.
165 li t0, SSTATUS_FS_INITIAL
202 /* Disable FPE usage in supervisor mode. */
203 li t0, SSTATUS_FS_MASK
210 * void cpu_throw(struct thread *old, struct thread *new)
213 /* Store the new curthread */
214 sd a1, PC_CURTHREAD(gp)
215 /* And the new pcb */
217 sd x13, PC_CURPCB(gp)
221 /* Switch to the new pmap */
222 ld t0, PCB_L1ADDR(x13)
223 srli t0, t0, PAGE_SHIFT
224 li t1, SATP_MODE_SV39
228 /* TODO: Invalidate the TLB */
238 ld s0, (PCB_S + 0 * 8)(x13)
239 ld s1, (PCB_S + 1 * 8)(x13)
240 ld s2, (PCB_S + 2 * 8)(x13)
241 ld s3, (PCB_S + 3 * 8)(x13)
242 ld s4, (PCB_S + 4 * 8)(x13)
243 ld s5, (PCB_S + 5 * 8)(x13)
244 ld s6, (PCB_S + 6 * 8)(x13)
245 ld s7, (PCB_S + 7 * 8)(x13)
246 ld s8, (PCB_S + 8 * 8)(x13)
247 ld s9, (PCB_S + 9 * 8)(x13)
248 ld s10, (PCB_S + 10 * 8)(x13)
249 ld s11, (PCB_S + 11 * 8)(x13)
252 /* Is FPE enabled for new thread? */
254 ld t1, (TF_SSTATUS)(t0)
255 li t2, SSTATUS_FS_MASK
257 beqz t3, 1f /* No, skip. */
259 /* Restore registers. */
268 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
273 * x3 to x7, x16 and x17 are caller saved
276 /* Store the new curthread */
277 sd a1, PC_CURTHREAD(gp)
278 /* And the new pcb */
280 sd x13, PC_CURPCB(gp)
282 /* Save the old context. */
285 /* Store ra, sp and the callee-saved registers */
291 sd s0, (PCB_S + 0 * 8)(x13)
292 sd s1, (PCB_S + 1 * 8)(x13)
293 sd s2, (PCB_S + 2 * 8)(x13)
294 sd s3, (PCB_S + 3 * 8)(x13)
295 sd s4, (PCB_S + 4 * 8)(x13)
296 sd s5, (PCB_S + 5 * 8)(x13)
297 sd s6, (PCB_S + 6 * 8)(x13)
298 sd s7, (PCB_S + 7 * 8)(x13)
299 sd s8, (PCB_S + 8 * 8)(x13)
300 sd s9, (PCB_S + 9 * 8)(x13)
301 sd s10, (PCB_S + 10 * 8)(x13)
302 sd s11, (PCB_S + 11 * 8)(x13)
306 * Is FPE enabled and is it in dirty state
307 * for the old thread?
310 ld t1, (TF_SSTATUS)(t0)
311 li t2, SSTATUS_FS_MASK
313 li t2, SSTATUS_FS_DIRTY
314 bne t3, t2, 1f /* No, skip. */
316 /* Yes, mark FPE state clean and save registers. */
317 li t2, ~SSTATUS_FS_MASK
319 li t2, SSTATUS_FS_CLEAN
321 sd t3, (TF_SSTATUS)(t0)
328 * Restore the saved context.
333 * TODO: We may need to flush the cache here if switching
339 /* Switch to the new pmap */
340 ld t0, PCB_L1ADDR(x13)
341 srli t0, t0, PAGE_SHIFT
342 li t1, SATP_MODE_SV39
346 /* TODO: Invalidate the TLB */
350 /* Release the old thread */
352 #if defined(SCHED_ULE) && defined(SMP)
353 /* Spin if TD_LOCK points to a blocked_lock */
354 la a2, _C_LABEL(blocked_lock)
360 /* Restore the registers */
366 ld s0, (PCB_S + 0 * 8)(x13)
367 ld s1, (PCB_S + 1 * 8)(x13)
368 ld s2, (PCB_S + 2 * 8)(x13)
369 ld s3, (PCB_S + 3 * 8)(x13)
370 ld s4, (PCB_S + 4 * 8)(x13)
371 ld s5, (PCB_S + 5 * 8)(x13)
372 ld s6, (PCB_S + 6 * 8)(x13)
373 ld s7, (PCB_S + 7 * 8)(x13)
374 ld s8, (PCB_S + 8 * 8)(x13)
375 ld s9, (PCB_S + 9 * 8)(x13)
376 ld s10, (PCB_S + 10 * 8)(x13)
377 ld s11, (PCB_S + 11 * 8)(x13)
380 /* Is FPE enabled for new thread? */
382 ld t1, (TF_SSTATUS)(t0)
383 li t2, SSTATUS_FS_MASK
385 beqz t3, 1f /* No, skip. */
387 /* Restore registers. */
393 .Lcpu_switch_panic_str:
394 .asciz "cpu_switch: %p\0"
398 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
399 * struct trapframe *frame)
402 ENTRY(fork_trampoline)
406 call _C_LABEL(fork_exit)
408 /* Restore sstatus */
409 ld t0, (TF_SSTATUS)(sp)
410 /* Ensure interrupts disabled */
415 /* Restore exception program counter */
419 /* Restore the registers */
420 ld t0, (TF_T + 0 * 8)(sp)
421 ld t1, (TF_T + 1 * 8)(sp)
422 ld t2, (TF_T + 2 * 8)(sp)
423 ld t3, (TF_T + 3 * 8)(sp)
424 ld t4, (TF_T + 4 * 8)(sp)
425 ld t5, (TF_T + 5 * 8)(sp)
426 ld t6, (TF_T + 6 * 8)(sp)
428 ld s0, (TF_S + 0 * 8)(sp)
429 ld s1, (TF_S + 1 * 8)(sp)
430 ld s2, (TF_S + 2 * 8)(sp)
431 ld s3, (TF_S + 3 * 8)(sp)
432 ld s4, (TF_S + 4 * 8)(sp)
433 ld s5, (TF_S + 5 * 8)(sp)
434 ld s6, (TF_S + 6 * 8)(sp)
435 ld s7, (TF_S + 7 * 8)(sp)
436 ld s8, (TF_S + 8 * 8)(sp)
437 ld s9, (TF_S + 9 * 8)(sp)
438 ld s10, (TF_S + 10 * 8)(sp)
439 ld s11, (TF_S + 11 * 8)(sp)
441 ld a0, (TF_A + 0 * 8)(sp)
442 ld a1, (TF_A + 1 * 8)(sp)
443 ld a2, (TF_A + 2 * 8)(sp)
444 ld a3, (TF_A + 3 * 8)(sp)
445 ld a4, (TF_A + 4 * 8)(sp)
446 ld a5, (TF_A + 5 * 8)(sp)
447 ld a6, (TF_A + 6 * 8)(sp)
448 ld a7, (TF_A + 7 * 8)(sp)
450 /* Load user ra and sp */
454 * Store our pcpup on stack, we will load it back
455 * on kernel mode trap.
460 /* Save kernel stack so we can use it doing a user trap */
464 /* Load user stack */
465 ld sp, (TF_SP - TF_SIZE)(sp)
471 /* Store ra, sp and the callee-saved registers */
477 sd s0, (PCB_S + 0 * 8)(a0)
478 sd s1, (PCB_S + 1 * 8)(a0)
479 sd s2, (PCB_S + 2 * 8)(a0)
480 sd s3, (PCB_S + 3 * 8)(a0)
481 sd s4, (PCB_S + 4 * 8)(a0)
482 sd s5, (PCB_S + 5 * 8)(a0)
483 sd s6, (PCB_S + 6 * 8)(a0)
484 sd s7, (PCB_S + 7 * 8)(a0)
485 sd s8, (PCB_S + 8 * 8)(a0)
486 sd s9, (PCB_S + 9 * 8)(a0)
487 sd s10, (PCB_S + 10 * 8)(a0)
488 sd s11, (PCB_S + 11 * 8)(a0)