2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include "opt_sched.h"
38 #include <machine/param.h>
39 #include <machine/asm.h>
40 #include <machine/riscvreg.h>
41 #include <machine/pte.h>
43 __FBSDID("$FreeBSD$");
46 .macro __fpe_state_save p
48 * Enable FPE usage in supervisor mode,
49 * so we can access registers.
51 li t0, SSTATUS_FS_INITIAL
57 fsd f0, (PCB_X + 0 * 16)(\p)
58 fsd f1, (PCB_X + 1 * 16)(\p)
59 fsd f2, (PCB_X + 2 * 16)(\p)
60 fsd f3, (PCB_X + 3 * 16)(\p)
61 fsd f4, (PCB_X + 4 * 16)(\p)
62 fsd f5, (PCB_X + 5 * 16)(\p)
63 fsd f6, (PCB_X + 6 * 16)(\p)
64 fsd f7, (PCB_X + 7 * 16)(\p)
65 fsd f8, (PCB_X + 8 * 16)(\p)
66 fsd f9, (PCB_X + 9 * 16)(\p)
67 fsd f10, (PCB_X + 10 * 16)(\p)
68 fsd f11, (PCB_X + 11 * 16)(\p)
69 fsd f12, (PCB_X + 12 * 16)(\p)
70 fsd f13, (PCB_X + 13 * 16)(\p)
71 fsd f14, (PCB_X + 14 * 16)(\p)
72 fsd f15, (PCB_X + 15 * 16)(\p)
73 fsd f16, (PCB_X + 16 * 16)(\p)
74 fsd f17, (PCB_X + 17 * 16)(\p)
75 fsd f18, (PCB_X + 18 * 16)(\p)
76 fsd f19, (PCB_X + 19 * 16)(\p)
77 fsd f20, (PCB_X + 20 * 16)(\p)
78 fsd f21, (PCB_X + 21 * 16)(\p)
79 fsd f22, (PCB_X + 22 * 16)(\p)
80 fsd f23, (PCB_X + 23 * 16)(\p)
81 fsd f24, (PCB_X + 24 * 16)(\p)
82 fsd f25, (PCB_X + 25 * 16)(\p)
83 fsd f26, (PCB_X + 26 * 16)(\p)
84 fsd f27, (PCB_X + 27 * 16)(\p)
85 fsd f28, (PCB_X + 28 * 16)(\p)
86 fsd f29, (PCB_X + 29 * 16)(\p)
87 fsd f30, (PCB_X + 30 * 16)(\p)
88 fsd f31, (PCB_X + 31 * 16)(\p)
90 /* Disable FPE usage in supervisor mode. */
91 li t0, SSTATUS_FS_MASK
95 .macro __fpe_state_load p
97 * Enable FPE usage in supervisor mode,
98 * so we can access registers.
100 li t0, SSTATUS_FS_INITIAL
103 /* Restore registers */
104 ld t0, (PCB_FCSR)(\p)
106 fld f0, (PCB_X + 0 * 16)(\p)
107 fld f1, (PCB_X + 1 * 16)(\p)
108 fld f2, (PCB_X + 2 * 16)(\p)
109 fld f3, (PCB_X + 3 * 16)(\p)
110 fld f4, (PCB_X + 4 * 16)(\p)
111 fld f5, (PCB_X + 5 * 16)(\p)
112 fld f6, (PCB_X + 6 * 16)(\p)
113 fld f7, (PCB_X + 7 * 16)(\p)
114 fld f8, (PCB_X + 8 * 16)(\p)
115 fld f9, (PCB_X + 9 * 16)(\p)
116 fld f10, (PCB_X + 10 * 16)(\p)
117 fld f11, (PCB_X + 11 * 16)(\p)
118 fld f12, (PCB_X + 12 * 16)(\p)
119 fld f13, (PCB_X + 13 * 16)(\p)
120 fld f14, (PCB_X + 14 * 16)(\p)
121 fld f15, (PCB_X + 15 * 16)(\p)
122 fld f16, (PCB_X + 16 * 16)(\p)
123 fld f17, (PCB_X + 17 * 16)(\p)
124 fld f18, (PCB_X + 18 * 16)(\p)
125 fld f19, (PCB_X + 19 * 16)(\p)
126 fld f20, (PCB_X + 20 * 16)(\p)
127 fld f21, (PCB_X + 21 * 16)(\p)
128 fld f22, (PCB_X + 22 * 16)(\p)
129 fld f23, (PCB_X + 23 * 16)(\p)
130 fld f24, (PCB_X + 24 * 16)(\p)
131 fld f25, (PCB_X + 25 * 16)(\p)
132 fld f26, (PCB_X + 26 * 16)(\p)
133 fld f27, (PCB_X + 27 * 16)(\p)
134 fld f28, (PCB_X + 28 * 16)(\p)
135 fld f29, (PCB_X + 29 * 16)(\p)
136 fld f30, (PCB_X + 30 * 16)(\p)
137 fld f31, (PCB_X + 31 * 16)(\p)
139 /* Disable FPE usage in supervisor mode. */
140 li t0, SSTATUS_FS_MASK
146 * fpe_state_save(struct thread *td)
148 ENTRY(fpe_state_save)
149 /* Get pointer to PCB */
157 * fpe_state_clear(void)
159 ENTRY(fpe_state_clear)
161 * Enable FPE usage in supervisor mode,
162 * so we can access registers.
164 li t0, SSTATUS_FS_INITIAL
201 /* Disable FPE usage in supervisor mode. */
202 li t0, SSTATUS_FS_MASK
210 * void cpu_throw(struct thread *old __unused, struct thread *new)
213 /* Activate the new thread's pmap. */
216 call _C_LABEL(pmap_activate_sw)
219 /* Store the new curthread */
220 sd a0, PC_CURTHREAD(tp)
221 /* And the new pcb */
223 sd x13, PC_CURPCB(tp)
230 ld s0, (PCB_S + 0 * 8)(x13)
231 ld s1, (PCB_S + 1 * 8)(x13)
232 ld s2, (PCB_S + 2 * 8)(x13)
233 ld s3, (PCB_S + 3 * 8)(x13)
234 ld s4, (PCB_S + 4 * 8)(x13)
235 ld s5, (PCB_S + 5 * 8)(x13)
236 ld s6, (PCB_S + 6 * 8)(x13)
237 ld s7, (PCB_S + 7 * 8)(x13)
238 ld s8, (PCB_S + 8 * 8)(x13)
239 ld s9, (PCB_S + 9 * 8)(x13)
240 ld s10, (PCB_S + 10 * 8)(x13)
241 ld s11, (PCB_S + 11 * 8)(x13)
244 /* Is FPE enabled for new thread? */
246 ld t1, (TF_SSTATUS)(t0)
247 li t2, SSTATUS_FS_MASK
249 beqz t3, 1f /* No, skip. */
251 /* Restore registers. */
260 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
265 * x3 to x7, x16 and x17 are caller saved
268 /* Store the new curthread */
269 sd a1, PC_CURTHREAD(tp)
270 /* And the new pcb */
272 sd x13, PC_CURPCB(tp)
274 /* Save the old context. */
277 /* Store ra, sp and the callee-saved registers */
282 sd s0, (PCB_S + 0 * 8)(x13)
283 sd s1, (PCB_S + 1 * 8)(x13)
284 sd s2, (PCB_S + 2 * 8)(x13)
285 sd s3, (PCB_S + 3 * 8)(x13)
286 sd s4, (PCB_S + 4 * 8)(x13)
287 sd s5, (PCB_S + 5 * 8)(x13)
288 sd s6, (PCB_S + 6 * 8)(x13)
289 sd s7, (PCB_S + 7 * 8)(x13)
290 sd s8, (PCB_S + 8 * 8)(x13)
291 sd s9, (PCB_S + 9 * 8)(x13)
292 sd s10, (PCB_S + 10 * 8)(x13)
293 sd s11, (PCB_S + 11 * 8)(x13)
297 * Is FPE enabled and is it in dirty state
298 * for the old thread?
301 ld t1, (TF_SSTATUS)(t0)
302 li t2, SSTATUS_FS_MASK
304 li t2, SSTATUS_FS_DIRTY
305 bne t3, t2, 1f /* No, skip. */
307 /* Yes, mark FPE state clean and save registers. */
308 li t2, ~SSTATUS_FS_MASK
310 li t2, SSTATUS_FS_CLEAN
312 sd t3, (TF_SSTATUS)(t0)
318 /* Activate the new thread's pmap */
323 call _C_LABEL(pmap_activate_sw)
326 /* Release the old thread */
328 #if defined(SCHED_ULE) && defined(SMP)
329 /* Spin if TD_LOCK points to a blocked_lock */
330 la s2, _C_LABEL(blocked_lock)
336 * Restore the saved context.
340 /* Restore the registers */
345 ld s0, (PCB_S + 0 * 8)(x13)
346 ld s1, (PCB_S + 1 * 8)(x13)
347 ld s2, (PCB_S + 2 * 8)(x13)
348 ld s3, (PCB_S + 3 * 8)(x13)
349 ld s4, (PCB_S + 4 * 8)(x13)
350 ld s5, (PCB_S + 5 * 8)(x13)
351 ld s6, (PCB_S + 6 * 8)(x13)
352 ld s7, (PCB_S + 7 * 8)(x13)
353 ld s8, (PCB_S + 8 * 8)(x13)
354 ld s9, (PCB_S + 9 * 8)(x13)
355 ld s10, (PCB_S + 10 * 8)(x13)
356 ld s11, (PCB_S + 11 * 8)(x13)
359 /* Is FPE enabled for new thread? */
361 ld t1, (TF_SSTATUS)(t0)
362 li t2, SSTATUS_FS_MASK
364 beqz t3, 1f /* No, skip. */
366 /* Restore registers. */
372 .Lcpu_switch_panic_str:
373 .asciz "cpu_switch: %p\0"
377 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
378 * struct trapframe *frame)
381 ENTRY(fork_trampoline)
385 call _C_LABEL(fork_exit)
387 /* Restore sstatus */
388 ld t0, (TF_SSTATUS)(sp)
389 /* Ensure interrupts disabled */
394 /* Restore exception program counter */
398 /* Restore the registers */
399 ld t0, (TF_T + 0 * 8)(sp)
400 ld t1, (TF_T + 1 * 8)(sp)
401 ld t2, (TF_T + 2 * 8)(sp)
402 ld t3, (TF_T + 3 * 8)(sp)
403 ld t4, (TF_T + 4 * 8)(sp)
404 ld t5, (TF_T + 5 * 8)(sp)
405 ld t6, (TF_T + 6 * 8)(sp)
407 ld s0, (TF_S + 0 * 8)(sp)
408 ld s1, (TF_S + 1 * 8)(sp)
409 ld s2, (TF_S + 2 * 8)(sp)
410 ld s3, (TF_S + 3 * 8)(sp)
411 ld s4, (TF_S + 4 * 8)(sp)
412 ld s5, (TF_S + 5 * 8)(sp)
413 ld s6, (TF_S + 6 * 8)(sp)
414 ld s7, (TF_S + 7 * 8)(sp)
415 ld s8, (TF_S + 8 * 8)(sp)
416 ld s9, (TF_S + 9 * 8)(sp)
417 ld s10, (TF_S + 10 * 8)(sp)
418 ld s11, (TF_S + 11 * 8)(sp)
420 ld a0, (TF_A + 0 * 8)(sp)
421 ld a1, (TF_A + 1 * 8)(sp)
422 ld a2, (TF_A + 2 * 8)(sp)
423 ld a3, (TF_A + 3 * 8)(sp)
424 ld a4, (TF_A + 4 * 8)(sp)
425 ld a5, (TF_A + 5 * 8)(sp)
426 ld a6, (TF_A + 6 * 8)(sp)
427 ld a7, (TF_A + 7 * 8)(sp)
429 /* Load user ra and gp */
434 * Store our pcpup on stack, we will load it back
435 * on kernel mode trap.
440 /* Save kernel stack so we can use it doing a user trap */
444 /* Load user stack */
445 ld sp, (TF_SP - TF_SIZE)(sp)
451 /* Store ra, sp and the callee-saved registers */
458 sd s0, (PCB_S + 0 * 8)(a0)
459 sd s1, (PCB_S + 1 * 8)(a0)
460 sd s2, (PCB_S + 2 * 8)(a0)
461 sd s3, (PCB_S + 3 * 8)(a0)
462 sd s4, (PCB_S + 4 * 8)(a0)
463 sd s5, (PCB_S + 5 * 8)(a0)
464 sd s6, (PCB_S + 6 * 8)(a0)
465 sd s7, (PCB_S + 7 * 8)(a0)
466 sd s8, (PCB_S + 8 * 8)(a0)
467 sd s9, (PCB_S + 9 * 8)(a0)
468 sd s10, (PCB_S + 10 * 8)(a0)
469 sd s11, (PCB_S + 11 * 8)(a0)