2 * Copyright (c) 1982, 1986 The Regents of the University of California.
3 * Copyright (c) 1989, 1990 William Jolitz
4 * Copyright (c) 1994 John Dyson
7 * This code is derived from software contributed to Berkeley by
8 * the Systems Programming Group of the University of Utah Computer
9 * Science Department, and William Jolitz.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 4. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91
36 * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
37 * from: src/sys/i386/i386/vm_machdep.c,v 1.132.2.2 2000/08/26 04:19:26 yokota
38 * JNPR: vm_machdep.c,v 1.8.2.2 2007/08/16 15:59:17 girish
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
44 #include "opt_compat.h"
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/malloc.h>
51 #include <sys/syscall.h>
52 #include <sys/sysent.h>
54 #include <sys/vnode.h>
55 #include <sys/vmmeter.h>
56 #include <sys/kernel.h>
57 #include <sys/sysctl.h>
58 #include <sys/unistd.h>
60 #include <machine/cache.h>
61 #include <machine/clock.h>
62 #include <machine/cpu.h>
63 #include <machine/md_var.h>
64 #include <machine/pcb.h>
67 #include <vm/vm_extern.h>
69 #include <vm/vm_kern.h>
70 #include <vm/vm_map.h>
71 #include <vm/vm_page.h>
72 #include <vm/vm_pageout.h>
73 #include <vm/vm_param.h>
75 #include <vm/uma_int.h>
80 #include <sys/sf_buf.h>
84 #define NSFBUFS (512 + maxusers * 16)
87 /* Duplicated from asm.h */
88 #if defined(__mips_o32)
93 #if defined(__mips_o32) || defined(__mips_o64)
94 #define CALLFRAME_SIZ (SZREG * (4 + 2))
95 #elif defined(__mips_n32) || defined(__mips_n64)
96 #define CALLFRAME_SIZ (SZREG * 4)
100 static void sf_buf_init(void *arg);
101 SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL);
104 * Expanded sf_freelist head. Really an SLIST_HEAD() in disguise, with the
105 * sf_freelist head with the sf_lock mutex.
108 SLIST_HEAD(, sf_buf) sf_head;
112 static u_int sf_buf_alloc_want;
116 * Finish a fork operation, with process p2 nearly set up.
117 * Copy and update the pcb, set up the stack so that the child
118 * ready to run and return to user mode.
121 cpu_fork(register struct thread *td1,register struct proc *p2,
122 struct thread *td2,int flags)
124 register struct proc *p1;
128 if ((flags & RFPROC) == 0)
130 /* It is assumed that the vm_thread_alloc called
131 * cpu_thread_alloc() before cpu_fork is called.
134 /* Point the pcb to the top of the stack */
137 /* Copy p1's pcb, note that in this case
138 * our pcb also includes the td_frame being copied
139 * too. The older mips2 code did an additional copy
140 * of the td_frame, for us that's not needed any
141 * longer (this copy does them both)
143 bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
145 /* Point mdproc and then copy over td1's contents
146 * md_proc is empty for MIPS
148 td2->td_md.md_flags = td1->td_md.md_flags & MDTD_FPUSED;
151 * Set up return-value registers as fork() libc stub expects.
153 td2->td_frame->v0 = 0;
154 td2->td_frame->v1 = 1;
155 td2->td_frame->a3 = 0;
157 if (td1 == PCPU_GET(fpcurthread))
158 MipsSaveCurFPState(td1);
160 pcb2->pcb_context[PCB_REG_RA] = (register_t)(intptr_t)fork_trampoline;
161 /* Make sp 64-bit aligned */
162 pcb2->pcb_context[PCB_REG_SP] = (register_t)(((vm_offset_t)td2->td_pcb &
163 ~(sizeof(__int64_t) - 1)) - CALLFRAME_SIZ);
164 pcb2->pcb_context[PCB_REG_S0] = (register_t)(intptr_t)fork_return;
165 pcb2->pcb_context[PCB_REG_S1] = (register_t)(intptr_t)td2;
166 pcb2->pcb_context[PCB_REG_S2] = (register_t)(intptr_t)td2->td_frame;
167 pcb2->pcb_context[PCB_REG_SR] = mips_rd_status() &
168 (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_INT_MASK);
170 * FREEBSD_DEVELOPERS_FIXME:
171 * Setup any other CPU-Specific registers (Not MIPS Standard)
172 * and/or bits in other standard MIPS registers (if CPU-Specific)
176 td2->td_md.md_tls = td1->td_md.md_tls;
177 td2->td_md.md_saved_intr = MIPS_SR_INT_IE;
178 td2->td_md.md_spinlock_count = 1;
180 if (td1->td_md.md_flags & MDTD_COP2USED) {
181 if (td1->td_md.md_cop2owner == COP2_OWNER_USERLAND) {
182 if (td1->td_md.md_ucop2)
183 octeon_cop2_save(td1->td_md.md_ucop2);
185 panic("cpu_fork: ucop2 is NULL but COP2 is enabled");
188 if (td1->td_md.md_cop2)
189 octeon_cop2_save(td1->td_md.md_cop2);
191 panic("cpu_fork: cop2 is NULL but COP2 is enabled");
195 if (td1->td_md.md_cop2) {
196 td2->td_md.md_cop2 = octeon_cop2_alloc_ctx();
197 memcpy(td2->td_md.md_cop2, td1->td_md.md_cop2,
198 sizeof(*td1->td_md.md_cop2));
200 if (td1->td_md.md_ucop2) {
201 td2->td_md.md_ucop2 = octeon_cop2_alloc_ctx();
202 memcpy(td2->td_md.md_ucop2, td1->td_md.md_ucop2,
203 sizeof(*td1->td_md.md_ucop2));
205 td2->td_md.md_cop2owner = td1->td_md.md_cop2owner;
206 pcb2->pcb_context[PCB_REG_SR] |= MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX;
207 /* Clear COP2 bits for userland & kernel */
208 td2->td_frame->sr &= ~MIPS_SR_COP_2_BIT;
209 pcb2->pcb_context[PCB_REG_SR] &= ~MIPS_SR_COP_2_BIT;
214 * Intercept the return address from a freshly forked process that has NOT
215 * been scheduled yet.
217 * This is needed to make kernel threads stay in kernel mode.
220 cpu_set_fork_handler(struct thread *td, void (*func) __P((void *)), void *arg)
223 * Note that the trap frame follows the args, so the function
224 * is really called like this: func(arg, frame);
226 td->td_pcb->pcb_context[PCB_REG_S0] = (register_t)(intptr_t)func;
227 td->td_pcb->pcb_context[PCB_REG_S1] = (register_t)(intptr_t)arg;
231 cpu_exit(struct thread *td)
236 cpu_thread_exit(struct thread *td)
239 if (PCPU_GET(fpcurthread) == td)
240 PCPU_GET(fpcurthread) = (struct thread *)0;
242 if (td->td_md.md_cop2)
243 memset(td->td_md.md_cop2, 0,
244 sizeof(*td->td_md.md_cop2));
245 if (td->td_md.md_ucop2)
246 memset(td->td_md.md_ucop2, 0,
247 sizeof(*td->td_md.md_ucop2));
252 cpu_thread_free(struct thread *td)
255 if (td->td_md.md_cop2)
256 octeon_cop2_free_ctx(td->td_md.md_cop2);
257 if (td->td_md.md_ucop2)
258 octeon_cop2_free_ctx(td->td_md.md_ucop2);
259 td->td_md.md_cop2 = NULL;
260 td->td_md.md_ucop2 = NULL;
265 cpu_thread_clean(struct thread *td)
270 cpu_thread_swapin(struct thread *td)
276 * The kstack may be at a different physical address now.
277 * Cache the PTEs for the Kernel stack in the machine dependent
278 * part of the thread struct so cpu_switch() can quickly map in
279 * the pcb struct and kernel stack.
281 for (i = 0; i < KSTACK_PAGES; i++) {
282 pte = pmap_pte(kernel_pmap, td->td_kstack + i * PAGE_SIZE);
283 td->td_md.md_upte[i] = *pte & ~TLBLO_SWBITS_MASK;
288 cpu_thread_swapout(struct thread *td)
293 cpu_thread_alloc(struct thread *td)
298 KASSERT((td->td_kstack & (1 << PAGE_SHIFT)) == 0, ("kernel stack must be aligned."));
299 td->td_pcb = (struct pcb *)(td->td_kstack +
300 td->td_kstack_pages * PAGE_SIZE) - 1;
301 td->td_frame = &td->td_pcb->pcb_regs;
303 for (i = 0; i < KSTACK_PAGES; i++) {
304 pte = pmap_pte(kernel_pmap, td->td_kstack + i * PAGE_SIZE);
305 td->td_md.md_upte[i] = *pte & ~TLBLO_SWBITS_MASK;
310 cpu_set_syscall_retval(struct thread *td, int error)
312 struct trapframe *locr0 = td->td_frame;
318 #if defined(__mips_n32) || defined(__mips_n64)
319 #ifdef COMPAT_FREEBSD32
320 if (code == SYS___syscall && SV_PROC_FLAG(td->td_proc, SV_ILP32))
324 if (code == SYS___syscall)
328 if (code == SYS_syscall)
330 else if (code == SYS___syscall) {
332 code = _QUAD_LOWWORD ? locr0->a1 : locr0->a0;
339 if (quad_syscall && code != SYS_lseek) {
341 * System call invoked through the
342 * SYS___syscall interface but the
343 * return value is really just 32
346 locr0->v0 = td->td_retval[0];
348 locr0->v1 = td->td_retval[0];
351 locr0->v0 = td->td_retval[0];
352 locr0->v1 = td->td_retval[1];
358 locr0->pc = td->td_pcb->pcb_tpc;
362 break; /* nothing to do */
365 if (quad_syscall && code != SYS_lseek) {
378 * Initialize machine state (pcb and trap frame) for a new thread about to
379 * upcall. Put enough state in the new thread's PCB to get it to go back
380 * userret(), where we can intercept it again to set the return (upcall)
381 * Address and stack, along with those from upcalls that are from other sources
382 * such as those generated in thread_userret() itself.
385 cpu_set_upcall(struct thread *td, struct thread *td0)
389 /* Point the pcb to the top of the stack. */
393 * Copy the upcall pcb. This loads kernel regs.
394 * Those not loaded individually below get their default
397 * XXXKSE It might be a good idea to simply skip this as
398 * the values of the other registers may be unimportant.
399 * This would remove any requirement for knowing the KSE
400 * at this time (see the matching comment below for
401 * more analysis) (need a good safe default).
402 * In MIPS, the trapframe is the first element of the PCB
403 * and gets copied when we copy the PCB. No separate copy
406 bcopy(td0->td_pcb, pcb2, sizeof(*pcb2));
409 * Set registers for trampoline to user mode.
412 pcb2->pcb_context[PCB_REG_RA] = (register_t)(intptr_t)fork_trampoline;
413 /* Make sp 64-bit aligned */
414 pcb2->pcb_context[PCB_REG_SP] = (register_t)(((vm_offset_t)td->td_pcb &
415 ~(sizeof(__int64_t) - 1)) - CALLFRAME_SIZ);
416 pcb2->pcb_context[PCB_REG_S0] = (register_t)(intptr_t)fork_return;
417 pcb2->pcb_context[PCB_REG_S1] = (register_t)(intptr_t)td;
418 pcb2->pcb_context[PCB_REG_S2] = (register_t)(intptr_t)td->td_frame;
419 /* Dont set IE bit in SR. sched lock release will take care of it */
420 pcb2->pcb_context[PCB_REG_SR] = mips_rd_status() &
421 (MIPS_SR_PX | MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_INT_MASK);
424 * FREEBSD_DEVELOPERS_FIXME:
425 * Setup any other CPU-Specific registers (Not MIPS Standard)
429 /* SMP Setup to release sched_lock in fork_exit(). */
430 td->td_md.md_spinlock_count = 1;
431 td->td_md.md_saved_intr = MIPS_SR_INT_IE;
433 /* Maybe we need to fix this? */
434 td->td_md.md_saved_sr = ( (MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT) |
435 (MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX) |
436 (MIPS_SR_INT_IE | MIPS_HARD_INT_MASK));
441 * Set that machine state for performing an upcall that has to
442 * be done in thread_userret() so that those upcalls generated
443 * in thread_userret() itself can be done as well.
446 cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
449 struct trapframe *tf;
453 * At the point where a function is called, sp must be 8
454 * byte aligned[for compatibility with 64-bit CPUs]
455 * in ``See MIPS Run'' by D. Sweetman, p. 269
458 sp = ((register_t)(intptr_t)(stack->ss_sp + stack->ss_size) & ~0x7) -
462 * Set the trap frame to point at the beginning of the uts
466 bzero(tf, sizeof(struct trapframe));
468 tf->pc = (register_t)(intptr_t)entry;
470 * MIPS ABI requires T9 to be the same as PC
471 * in subroutine entry point
473 tf->t9 = (register_t)(intptr_t)entry;
474 tf->a0 = (register_t)(intptr_t)arg;
477 * Keep interrupt mask
479 td->td_frame->sr = MIPS_SR_KSU_USER | MIPS_SR_EXL | MIPS_SR_INT_IE |
480 (mips_rd_status() & MIPS_SR_INT_MASK);
481 #if defined(__mips_n32)
482 td->td_frame->sr |= MIPS_SR_PX;
483 #elif defined(__mips_n64)
484 td->td_frame->sr |= MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX;
486 /* tf->sr |= (ALL_INT_MASK & idle_mask) | SR_INT_ENAB; */
487 /**XXX the above may now be wrong -- mips2 implements this as panic */
489 * FREEBSD_DEVELOPERS_FIXME:
490 * Setup any other CPU-Specific registers (Not MIPS Standard)
496 * Implement the pre-zeroed page mechanism.
497 * This routine is called from the idle loop.
500 #define ZIDLE_LO(v) ((v) * 2 / 3)
501 #define ZIDLE_HI(v) ((v) * 4 / 5)
504 * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
508 sf_buf_init(void *arg)
510 struct sf_buf *sf_bufs;
515 TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
517 mtx_init(&sf_freelist.sf_lock, "sf_bufs list lock", NULL, MTX_DEF);
518 SLIST_INIT(&sf_freelist.sf_head);
519 sf_base = kva_alloc(nsfbufs * PAGE_SIZE);
520 sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
522 for (i = 0; i < nsfbufs; i++) {
523 sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
524 SLIST_INSERT_HEAD(&sf_freelist.sf_head, &sf_bufs[i], free_list);
526 sf_buf_alloc_want = 0;
530 * Get an sf_buf from the freelist. Will block if none are available.
533 sf_buf_alloc(struct vm_page *m, int flags)
538 mtx_lock(&sf_freelist.sf_lock);
539 while ((sf = SLIST_FIRST(&sf_freelist.sf_head)) == NULL) {
540 if (flags & SFB_NOWAIT)
543 SFSTAT_INC(sf_allocwait);
544 error = msleep(&sf_freelist, &sf_freelist.sf_lock,
545 (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
549 * If we got a signal, don't risk going back to sleep.
555 SLIST_REMOVE_HEAD(&sf_freelist.sf_head, free_list);
558 nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
559 pmap_qenter(sf->kva, &sf->m, 1);
561 mtx_unlock(&sf_freelist.sf_lock);
566 * Release resources back to the system.
569 sf_buf_free(struct sf_buf *sf)
571 pmap_qremove(sf->kva, 1);
572 mtx_lock(&sf_freelist.sf_lock);
573 SLIST_INSERT_HEAD(&sf_freelist.sf_head, sf, free_list);
575 if (sf_buf_alloc_want > 0)
576 wakeup(&sf_freelist);
577 mtx_unlock(&sf_freelist.sf_lock);
579 #endif /* !__mips_n64 */
582 * Software interrupt handler for queued VM system processing.
588 if (busdma_swi_pending)
593 cpu_set_user_tls(struct thread *td, void *tls_base)
596 td->td_md.md_tls = (char*)tls_base;
604 #define DB_PRINT_REG(ptr, regname) \
605 db_printf(" %-12s %p\n", #regname, (void *)(intptr_t)((ptr)->regname))
607 #define DB_PRINT_REG_ARRAY(ptr, arrname, regname) \
608 db_printf(" %-12s %p\n", #regname, (void *)(intptr_t)((ptr)->arrname[regname]))
611 dump_trapframe(struct trapframe *trapframe)
614 db_printf("Trapframe at %p\n", trapframe);
616 DB_PRINT_REG(trapframe, zero);
617 DB_PRINT_REG(trapframe, ast);
618 DB_PRINT_REG(trapframe, v0);
619 DB_PRINT_REG(trapframe, v1);
620 DB_PRINT_REG(trapframe, a0);
621 DB_PRINT_REG(trapframe, a1);
622 DB_PRINT_REG(trapframe, a2);
623 DB_PRINT_REG(trapframe, a3);
624 #if defined(__mips_n32) || defined(__mips_n64)
625 DB_PRINT_REG(trapframe, a4);
626 DB_PRINT_REG(trapframe, a5);
627 DB_PRINT_REG(trapframe, a6);
628 DB_PRINT_REG(trapframe, a7);
629 DB_PRINT_REG(trapframe, t0);
630 DB_PRINT_REG(trapframe, t1);
631 DB_PRINT_REG(trapframe, t2);
632 DB_PRINT_REG(trapframe, t3);
634 DB_PRINT_REG(trapframe, t0);
635 DB_PRINT_REG(trapframe, t1);
636 DB_PRINT_REG(trapframe, t2);
637 DB_PRINT_REG(trapframe, t3);
638 DB_PRINT_REG(trapframe, t4);
639 DB_PRINT_REG(trapframe, t5);
640 DB_PRINT_REG(trapframe, t6);
641 DB_PRINT_REG(trapframe, t7);
643 DB_PRINT_REG(trapframe, s0);
644 DB_PRINT_REG(trapframe, s1);
645 DB_PRINT_REG(trapframe, s2);
646 DB_PRINT_REG(trapframe, s3);
647 DB_PRINT_REG(trapframe, s4);
648 DB_PRINT_REG(trapframe, s5);
649 DB_PRINT_REG(trapframe, s6);
650 DB_PRINT_REG(trapframe, s7);
651 DB_PRINT_REG(trapframe, t8);
652 DB_PRINT_REG(trapframe, t9);
653 DB_PRINT_REG(trapframe, k0);
654 DB_PRINT_REG(trapframe, k1);
655 DB_PRINT_REG(trapframe, gp);
656 DB_PRINT_REG(trapframe, sp);
657 DB_PRINT_REG(trapframe, s8);
658 DB_PRINT_REG(trapframe, ra);
659 DB_PRINT_REG(trapframe, sr);
660 DB_PRINT_REG(trapframe, mullo);
661 DB_PRINT_REG(trapframe, mulhi);
662 DB_PRINT_REG(trapframe, badvaddr);
663 DB_PRINT_REG(trapframe, cause);
664 DB_PRINT_REG(trapframe, pc);
667 DB_SHOW_COMMAND(pcb, ddb_dump_pcb)
671 struct trapframe *trapframe;
673 /* Determine which thread to examine. */
675 td = db_lookup_thread(addr, TRUE);
681 db_printf("Thread %d at %p\n", td->td_tid, td);
683 db_printf("PCB at %p\n", pcb);
685 trapframe = &pcb->pcb_regs;
686 dump_trapframe(trapframe);
688 db_printf("PCB Context:\n");
689 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S0);
690 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S1);
691 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S2);
692 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S3);
693 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S4);
694 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S5);
695 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S6);
696 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S7);
697 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_SP);
698 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_S8);
699 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_RA);
700 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_SR);
701 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_GP);
702 DB_PRINT_REG_ARRAY(pcb, pcb_context, PCB_REG_PC);
704 db_printf("PCB onfault = %p\n", pcb->pcb_onfault);
705 db_printf("md_saved_intr = 0x%0lx\n", (long)td->td_md.md_saved_intr);
706 db_printf("md_spinlock_count = %d\n", td->td_md.md_spinlock_count);
708 if (td->td_frame != trapframe) {
709 db_printf("td->td_frame %p is not the same as pcb_regs %p\n",
710 td->td_frame, trapframe);
715 * Dump the trapframe beginning at address specified by first argument.
717 DB_SHOW_COMMAND(trapframe, ddb_dump_trapframe)
723 dump_trapframe((struct trapframe *)addr);