2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2009 Neelkanth Natu
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/cpuset.h>
38 #include <sys/malloc.h>
39 #include <sys/mutex.h>
40 #include <sys/kernel.h>
43 #include <sys/sched.h>
48 #include <vm/vm_extern.h>
49 #include <vm/vm_kern.h>
51 #include <machine/clock.h>
52 #include <machine/smp.h>
53 #include <machine/hwfunc.h>
54 #include <machine/intr_machdep.h>
55 #include <machine/cache.h>
56 #include <machine/tlb.h>
58 struct pcb stoppcbs[MAXCPU];
61 static struct mtx ap_boot_mtx;
63 static volatile int aps_ready;
64 static volatile int mp_naps;
67 ipi_send(struct pcpu *pc, int ipi)
70 CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi);
72 atomic_set_32(&pc->pc_pending_ipis, ipi);
73 platform_ipi_send(pc->pc_cpuid);
75 CTR1(KTR_SMP, "%s: sent", __func__);
79 ipi_all_but_self(int ipi)
83 other_cpus = all_cpus;
84 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
85 ipi_selected(other_cpus, ipi);
88 /* Send an IPI to a set of cpus. */
90 ipi_selected(cpuset_t cpus, int ipi)
94 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
95 if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
96 CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
103 /* Send an IPI to a specific CPU. */
105 ipi_cpu(int cpu, u_int ipi)
108 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
109 ipi_send(cpuid_to_pcpu[cpu], ipi);
113 * Handle an IPI sent to this processor.
116 mips_ipi_handler(void *arg)
118 u_int cpu, ipi, ipi_bitmap;
121 cpu = PCPU_GET(cpuid);
123 platform_ipi_clear(); /* quiesce the pending ipi interrupt */
125 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
127 return (FILTER_STRAY);
129 CTR1(KTR_SMP, "smp_handle_ipi(), ipi_bitmap=%x", ipi_bitmap);
131 while ((bit = ffs(ipi_bitmap))) {
137 CTR0(KTR_SMP, "IPI_RENDEZVOUS");
138 smp_rendezvous_action();
142 CTR0(KTR_SMP, "IPI_AST");
147 * IPI_STOP_HARD is mapped to IPI_STOP so it is not
148 * necessary to add it in the switch.
150 CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD");
152 savectx(&stoppcbs[cpu]);
155 /* Indicate we are stopped */
156 CPU_SET_ATOMIC(cpu, &stopped_cpus);
158 /* Wait for restart */
159 while (!CPU_ISSET(cpu, &started_cpus))
162 CPU_CLR_ATOMIC(cpu, &started_cpus);
163 CPU_CLR_ATOMIC(cpu, &stopped_cpus);
164 CTR0(KTR_SMP, "IPI_STOP (restart)");
167 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
168 sched_preempt(curthread);
171 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
175 panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu);
179 return (FILTER_HANDLED);
188 dpcpu = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
192 if (platform_start_ap(cpuid) != 0)
193 return (-1); /* could not start AP */
195 for (ms = 0; ms < 5000; ++ms) {
197 return (0); /* success */
202 return (-2); /* timeout initializing AP */
206 cpu_mp_setmaxid(void)
211 platform_cpu_mask(&cpumask);
214 while ((cpu = CPU_FFS(&cpumask)) != 0) {
217 CPU_CLR(cpu, &cpumask);
223 mp_maxid = min(last, MAXCPU) - 1;
227 cpu_mp_announce(void)
235 return (platform_smp_topo());
242 return (mp_ncpus > 1);
251 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
254 platform_cpu_mask(&cpumask);
256 while (!CPU_EMPTY(&cpumask)) {
257 cpuid = CPU_FFS(&cpumask) - 1;
258 CPU_CLR(cpuid, &cpumask);
260 if (cpuid >= MAXCPU) {
261 printf("cpu_mp_start: ignoring AP #%d.\n", cpuid);
265 if (cpuid != platform_processor_id()) {
266 if ((error = start_ap(cpuid)) != 0) {
267 printf("AP #%d failed to start: %d\n", cpuid, error);
271 printf("AP #%d started!\n", cpuid);
273 CPU_SET(cpuid, &all_cpus);
278 smp_init_secondary(u_int32_t cpuid)
283 tlb_invalidate_all();
284 mips_wr_wired(VMWIRED_ENTRIES);
287 * We assume that the L1 cache on the APs is identical to the one
290 mips_dcache_wbinv_all();
291 mips_icache_sync_all();
297 pcpu_init(PCPU_ADDR(cpuid), cpuid, sizeof(struct pcpu));
298 dpcpu_init(dpcpu, cpuid);
300 /* The AP has initialized successfully - allow the BSP to proceed */
303 /* Spin until the BSP is ready to release the APs */
307 #ifdef PLATFORM_INIT_SECONDARY
308 platform_init_secondary(cpuid);
311 /* Initialize curthread. */
312 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
313 PCPU_SET(curthread, PCPU_GET(idlethread));
315 mtx_lock_spin(&ap_boot_mtx);
319 CTR1(KTR_SMP, "SMP: AP CPU #%d launched", PCPU_GET(cpuid));
322 printf("SMP: AP CPU #%d launched.\n", PCPU_GET(cpuid));
324 if (smp_cpus == mp_ncpus) {
325 atomic_store_rel_int(&smp_started, 1);
328 mtx_unlock_spin(&ap_boot_mtx);
330 while (smp_started == 0)
333 /* Start per-CPU event timers. */
336 /* enter the scheduler */
339 panic("scheduler returned us to %s", __func__);
344 release_aps(void *dummy __unused)
351 #ifdef PLATFORM_INIT_SECONDARY
352 platform_init_secondary(0);
358 ipi_irq = platform_ipi_hardintr_num();
360 cpu_establish_hardintr("ipi", mips_ipi_handler, NULL, NULL,
361 ipi_irq, INTR_TYPE_MISC | INTR_EXCL, NULL);
363 ipi_irq = platform_ipi_softintr_num();
364 cpu_establish_softintr("ipi", mips_ipi_handler, NULL, NULL,
365 ipi_irq, INTR_TYPE_MISC | INTR_EXCL, NULL);
368 atomic_store_rel_int(&aps_ready, 1);
370 while (smp_started == 0)
374 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);