4 #include "opt_kstack_pages.h"
12 #include <sys/malloc.h>
13 #include <sys/mutex.h>
14 #include <sys/kernel.h>
17 #include <sys/sysctl.h>
22 #include <vm/vm_map.h>
24 #include <machine/atomic.h>
25 #include <machine/clock.h>
26 #include <machine/md_var.h>
27 #include <machine/pcb.h>
28 #include <machine/pmap.h>
29 #include <machine/smp.h>
31 static struct mtx ap_boot_mtx;
32 extern struct pcpu __pcpu[];
33 extern int num_tlbentries;
34 void mips_start_timer(void);
35 static volatile int aps_ready = 0;
37 u_int32_t boot_cpu_id;
46 * To implement IPIs on MIPS CPU, we use the Interrupt Line 2 ( bit 4 of cause
47 * register) and a bitmap to avoid redundant IPI interrupts. To interrupt a
48 * set of CPUs, the sender routine runs in a ' loop ' sending interrupts to
49 * all the specified CPUs. A single Mutex (smp_ipi_mtx) is used for all IPIs
50 * that spinwait for delivery. This includes the following IPIs
58 * send an IPI to a set of cpus.
61 ipi_selected(u_int32_t cpus, u_int ipi)
64 u_int cpuid, new_pending, old_pending;
66 CTR3(KTR_SMP, "%s: cpus: %x, ipi: %x\n", __func__, cpus, ipi);
68 while ((cpuid = ffs(cpus)) != 0) {
70 cpus &= ~(1 << cpuid);
71 pcpu = pcpu_find(cpuid);
75 old_pending = pcpu->pc_pending_ipis;
76 new_pending = old_pending | ipi;
77 } while (!atomic_cmpset_int(&pcpu->pc_pending_ipis,
78 old_pending, new_pending));
83 mips_ipi_send (cpuid);
89 * send an IPI to all CPUs EXCEPT myself
92 ipi_all_but_self(u_int ipi)
95 ipi_selected(PCPU_GET(other_cpus), ipi);
99 * Handle an IPI sent to this processor.
102 smp_handle_ipi(struct trapframe *frame)
104 cpumask_t cpumask; /* This cpu mask */
105 u_int ipi, ipi_bitmap;
107 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
108 cpumask = PCPU_GET(cpumask);
110 CTR1(KTR_SMP, "smp_handle_ipi(), ipi_bitmap=%x", ipi_bitmap);
113 * Find the lowest set bit.
115 ipi = ipi_bitmap & ~(ipi_bitmap - 1);
119 CTR0(KTR_SMP, "IPI_INVLTLB");
123 CTR0(KTR_SMP, "IPI_RENDEZVOUS");
124 smp_rendezvous_action();
128 CTR0(KTR_SMP, "IPI_AST");
132 CTR0(KTR_SMP, "IPI_STOP");
133 atomic_set_int(&stopped_cpus, cpumask);
135 while ((started_cpus & cpumask) == 0)
137 atomic_clear_int(&started_cpus, cpumask);
138 atomic_clear_int(&stopped_cpus, cpumask);
146 cpu_mp_setmaxid(void)
149 mp_maxid = MAXCPU - 1;
153 smp_init_secondary(u_int32_t cpuid)
157 panic ("cpu id exceeds MAXCPU\n");
161 R4K_TLBFlush(num_tlbentries);
162 R4K_SetWIRED(VMWIRED_ENTRIES);
167 mips_cp0_status_write(0);
171 mips_sync(); mips_sync();
172 /* Initialize curthread. */
173 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
174 PCPU_SET(curthread, PCPU_GET(idlethread));
176 mtx_lock_spin(&ap_boot_mtx);
180 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
182 /* Build our map of 'other' CPUs. */
183 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
185 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
187 if (smp_cpus == mp_ncpus) {
192 mtx_unlock_spin(&ap_boot_mtx);
194 while (smp_started == 0)
196 /* Enable Interrupt */
197 mips_cp0_status_write(SR_INT_ENAB);
198 /* ok, now grab sched_lock and enter the scheduler */
199 mtx_lock_spin(&sched_lock);
202 * Correct spinlock nesting. The idle thread context that we are
203 * borrowing was created so that it would start out with a single
204 * spin lock (sched_lock) held in fork_trampoline(). Since we've
205 * explicitly acquired locks in this function, the nesting count
206 * is now 2 rather than 1. Since we are nested, calling
207 * spinlock_exit() will simply adjust the counts without allowing
208 * spin lock using code to interrupt us.
211 KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
213 binuptime(PCPU_PTR(switchtime));
214 PCPU_SET(switchticks, ticks);
216 /* kick off the clock on this cpu */
218 cpu_throw(NULL, choosethread()); /* doesn't return */
220 panic("scheduler returned us to %s", __func__);
224 smp_start_secondary(int cpuid)
231 printf("smp_start_secondary: starting cpu %d\n", cpuid);
233 dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
234 pcpu_init(&__pcpu[cpuid], cpuid, sizeof(struct pcpu));
235 dpcpu_init(dpcpu, cpuid);
238 printf("smp_start_secondary: cpu %d started\n", cpuid);
248 /* XXX: Need to check for valid platforms here. */
250 boot_cpu_id = PCPU_GET(cpuid);
251 KASSERT(boot_cpu_id == 0, ("cpu_mp_probe() called on non-primary CPU"));
252 all_cpus = PCPU_GET(cpumask);
255 /* Make sure we have at least one secondary CPU. */
257 for (i = 0; i < MAXCPU; i++) {
268 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
271 for (i = 0; i < MAXCPU; i++) {
273 if (i == boot_cpu_id)
275 if (smp_start_secondary(i)) {
276 all_cpus |= (1 << cpuid);
281 idle_mask |= CR_INT_IPI;
282 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
286 release_aps(void *dummy __unused)
288 if (bootverbose && mp_ncpus > 1)
289 printf("%s: releasing secondary CPUs\n", __func__);
290 atomic_store_rel_int(&aps_ready, 1);
292 while (mp_ncpus > 1 && smp_started == 0)
296 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);