2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. Berkeley Software Design Inc's name may not be used to endorse or
13 * promote products derived from this software without specific prior
16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp
31 * Copyright (c) 2002 Jake Burkholder.
32 * Copyright (c) 2006 Kip Macy <kmacy@FreeBSD.org>.
33 * All rights reserved.
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
47 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57 #include <sys/cdefs.h>
58 __FBSDID("$FreeBSD$");
60 #include "opt_trap_trace.h"
62 #include <sys/param.h>
63 #include <sys/systm.h>
66 #include <sys/kernel.h>
68 #include <sys/mutex.h>
71 #include <sys/sched.h>
75 #include <vm/vm_param.h>
77 #include <vm/vm_kern.h>
78 #include <vm/vm_extern.h>
79 #include <vm/vm_map.h>
81 #include <dev/ofw/openfirm.h>
83 #include <machine/asi.h>
84 #include <machine/atomic.h>
85 #include <machine/bus.h>
86 #include <machine/md_var.h>
87 #include <machine/metadata.h>
88 #include <machine/ofw_machdep.h>
89 #include <machine/pcb.h>
90 #include <machine/smp.h>
91 #include <machine/tick.h>
92 #include <machine/pstate.h>
93 #include <machine/tlb.h>
94 #include <machine/tte.h>
95 #include <machine/tte_hash.h>
96 #include <machine/tsb.h>
97 #include <machine/trap.h>
98 #include <machine/hypervisorvar.h>
99 #include <machine/hv_api.h>
100 #include <machine/asm.h>
103 * Argument area used to pass data to non-boot processors as they start up.
104 * This must be statically initialized with a known invalid cpuid,
107 struct cpu_start_args cpu_start_args = { 0, -1, 0, -1 };
108 struct ipi_cache_args ipi_cache_args;
109 struct ipi_tlb_args ipi_tlb_args;
110 struct pcb stoppcbs[MAXCPU];
114 vm_offset_t mp_tramp;
118 static volatile cpumask_t shutdown_cpus;
120 void cpu_mp_unleash(void *);
121 SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
124 #ifndef TRAP_TRACE_ENTRIES
125 #define TRAP_TRACE_ENTRIES 64
127 extern trap_trace_entry_t trap_trace_entry[MAXCPU][TRAP_TRACE_ENTRIES];
130 mp_trap_trace_init(void)
134 printf("curcpu %d trap_trace_entry %p TRAP_TRACE_ENTRIES %d\n", curcpu, &trap_trace_entry[curcpu][0], TRAP_TRACE_ENTRIES);
136 /* Configure the trap trace buffer for the current CPU. */
137 if ((ret = hv_ttrace_buf_conf((uint64_t) vtophys(&trap_trace_entry[curcpu][0]),
138 (uint64_t) TRAP_TRACE_ENTRIES, &ret1)) != 0)
139 printf("%s: hv_ttrace_buf_conf error %lu\n", __FUNCTION__, ret);
141 /* Enable trap tracing for the current CPU. */
142 else if ((ret = hv_ttrace_enable((uint64_t) -1, &ret1)) != 0)
143 printf("%s: hv_ttrace_enable error %lu\n", __FUNCTION__, ret);
146 void trap_trace_report(int);
148 static int trace_trap_lock;
151 trap_trace_report(int cpuid)
155 while (!atomic_cmpset_acq_int(&trace_trap_lock, 0, 1))
158 for (i = 0; i < MAXCPU; i++) {
159 if (cpuid != -1 && cpuid != i)
162 for (j = 0; j < TRAP_TRACE_ENTRIES; j++) {
163 trap_trace_entry_t *p = &trap_trace_entry[i][j];
165 printf("0x%08jx [%02d][%04d] tpc 0x%jx type 0x%x hpstat 0x%x tl %u gl %u tt 0x%hx tag 0x%hx tstate 0x%jx f1 0x%jx f2 0x%jx f3 0x%jx f4 0x%jx\n",
166 p->tte_tick, i, j, p->tte_tpc,p->tte_type,p->tte_hpstat,
167 p->tte_tl,p->tte_gl,p->tte_tt,p->tte_tag,p->tte_tstate,
168 p->tte_f1,p->tte_f2,p->tte_f3,p->tte_f4);
172 atomic_store_rel_int(&trace_trap_lock, 0);
182 v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE);
184 panic("mp_tramp_alloc");
185 bcopy(mp_tramp_code, v, mp_tramp_code_len);
187 *(u_long *)(v + mp_tramp_func) = (u_long)mp_startup;
189 for (i = 0; i < PAGE_SIZE; i += sizeof(long)*4 /* XXX L1 cacheline size */)
191 return (vm_offset_t)v;
195 mp_set_tsb_desc_ra(vm_paddr_t tsb_desc_ra)
197 *(u_long *)(mp_tramp + mp_tramp_tsb_desc_ra) = tsb_desc_ra;
201 mp_add_nucleus_mapping(vm_offset_t va, tte_t tte_data)
206 entry = (uint64_t *)(mp_tramp + mp_tramp_code_len + slot*sizeof(*entry)*2);
208 *(entry + 1) = tte_data;
209 *(uint64_t *)(mp_tramp + mp_tramp_tte_slots) = slot + 1;
214 * Probe for other cpus.
217 cpu_mp_setmaxid(void)
224 all_cpus = 1 << PCPU_GET(cpuid);
229 for (child = OF_child(root); child != 0; child = OF_peer(child)) {
230 if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 &&
231 strcmp(buf, "cpu") == 0)
241 return (mp_maxid > 0);
248 return smp_topo_none();
252 start_ap_bycpuid(int cpuid, void *func, u_long arg)
263 (cell_t)"SUNW,start-cpu-by-cpuid",
273 args.func = (cell_t)func;
274 args.arg = (cell_t)arg;
276 return (int)args.result;
281 * Fire up any non-boot processors.
286 volatile struct cpu_start_args *csa;
293 int cpuid, bp_skipped;
297 csa = &cpu_start_args;
298 clock = cpuid = bp_skipped = 0;
299 for (child = OF_child(root); child != 0; child = OF_peer(child)) {
300 if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 ||
301 strcmp(buf, "cpu") != 0)
303 /* skip boot processor */
310 if (OF_getprop(child, "clock-frequency", &clock,
312 panic("cpu_mp_start: can't get clock");
315 start_ap_bycpuid(cpuid, (void *)mp_tramp, (uint64_t)cpuid);
317 while (csa->csa_state != CPU_INIT)
320 mp_ncpus = cpuid + 1;
322 cpu_identify(0, clock, cpuid);
324 va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
325 pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
326 pcpu_init(pc, cpuid, sizeof(*pc));
327 dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
331 all_cpus |= 1 << cpuid;
333 if (mp_ncpus == MAXCPU)
336 printf("%d cpus: UltraSparc T1 Processor (%d.%02d MHz CPU)\n", mp_ncpus,
337 (clock + 4999) / 1000000, ((clock + 4999) / 10000) % 100);
339 PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid)));
344 cpu_mp_announce(void)
349 cpu_mp_unleash(void *v)
351 volatile struct cpu_start_args *csa;
355 csa = &cpu_start_args;
356 csa->csa_count = mp_ncpus;
357 printf("mp_ncpus=%d\n", mp_ncpus);
358 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
359 if (pc->pc_cpuid == PCPU_GET(cpuid))
362 KASSERT(pc->pc_idlethread != NULL,
363 ("cpu_mp_unleash: idlethread is NULL"));
364 pc->pc_curthread = pc->pc_idlethread;
365 pc->pc_curpcb = pc->pc_curthread->td_pcb;
366 pc->pc_curpmap = kernel_pmap;
368 csa->csa_pcpu = TLB_PHYS_TO_DIRECT(vtophys(pc->pc_addr));
370 /* allow AP to run */
371 csa->csa_cpuid = pc->pc_cpuid;
374 while (csa->csa_state != CPU_BOOTSTRAP)
385 cpu_mp_bootstrap(struct pcpu *pc)
387 volatile struct cpu_start_args *csa;
389 csa = &cpu_start_args;
391 tsb_set_scratchpad_kernel(&kernel_pmap->pm_tsb);
392 tte_hash_set_scratchpad_kernel(kernel_pmap->pm_hash);
398 mp_trap_trace_init();
402 * enable interrupts now that we have our trap table set
404 intr_restore_all(PSTATE_KERNEL);
407 KASSERT(curthread != NULL, ("cpu_mp_bootstrap: curthread"));
408 PCPU_SET(other_cpus, all_cpus & ~(1 << curcpu));
409 printf("AP: #%d\n", curcpu);
412 csa->csa_state = CPU_BOOTSTRAP;
414 while (csa->csa_count != 0)
416 /* ok, now enter the scheduler */
421 cpu_mp_shutdown(void)
426 shutdown_cpus = PCPU_GET(other_cpus);
427 if (stopped_cpus != PCPU_GET(other_cpus)) /* XXX */
428 stop_cpus(stopped_cpus ^ PCPU_GET(other_cpus));
430 while (shutdown_cpus != 0) {
432 printf("timeout shutting down CPUs.\n");
436 /* XXX: delay a bit to allow the CPUs to actually enter the PROM. */
442 cpu_ipi_ast(struct trapframe *tf)
447 cpu_ipi_stop(struct trapframe *tf)
450 CTR1(KTR_SMP, "cpu_ipi_stop: stopped %d", curcpu);
451 savectx(&stoppcbs[curcpu]);
452 atomic_set_acq_int(&stopped_cpus, PCPU_GET(cpumask));
453 while ((started_cpus & PCPU_GET(cpumask)) == 0) {
454 if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) {
455 atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask));
458 atomic_clear_rel_int(&started_cpus, PCPU_GET(cpumask));
459 atomic_clear_rel_int(&stopped_cpus, PCPU_GET(cpumask));
460 CTR1(KTR_SMP, "cpu_ipi_stop: restarted %d", curcpu);
464 cpu_ipi_preempt(struct trapframe *tf)
466 sched_preempt(curthread);
470 cpu_ipi_selected(int cpu_count, uint16_t *cpulist, u_long d0, u_long d1, u_long d2, uint64_t *ackmask)
475 init_mondo(d0, d1, d2, (uint64_t)pmap_kextract((vm_offset_t)ackmask));
481 int error, new_cpu_count;
482 vm_paddr_t cpulist_ra;
484 cpulist_ra = TLB_DIRECT_TO_PHYS((vm_offset_t)cpulist);
485 if ((error = hv_cpu_mondo_send(cpu_count, cpulist_ra)) == H_EWOULDBLOCK) {
487 for (i = 0; i < cpu_count; i++) {
488 if (cpulist[i] != 0xffff)
489 cpulist[new_cpu_count++] = cpulist[i];
491 cpu_count = new_cpu_count;
493 if (cpu_count == 0) {
494 printf("no more cpus to send to but mondo_send returned EWOULDBLOCK\n");
497 if ((retries & 0x1) == 0x1)
503 printf("used up retries - cpus remaining: %d - cpus: ",
505 for (i = 0; i < cpu_count; i++)
506 printf("#%d ", cpulist[i]);
510 if (error == H_ENOCPU) {
511 printf("bad cpuid: ");
512 for (i = 0; i < cpu_count; i++)
513 printf("#%d ", cpulist[i]);
517 panic("can't handle error %d from cpu_mondo_send\n", error);
522 ipi_selected(cpumask_t icpus, u_int ipi)
531 * 3) forward_wakeup appears to abuse ASTs
532 * 4) handling 4-way threading vs 2-way threading should happen here
533 * and not in forward wakeup
535 cpulist = PCPU_GET(cpulist);
536 cpus = (icpus & ~PCPU_GET(cpumask));
538 for (cpu_count = 0, i = 0; i < 32 && cpus; cpus = cpus >> 1, i++) {
542 cpulist[cpu_count] = (uint16_t)i;
546 cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
551 ipi_cpu(int cpu, u_int ipi)
559 * 3) forward_wakeup appears to abuse ASTs
560 * 4) handling 4-way threading vs 2-way threading should happen here
561 * and not in forward wakeup
563 cpulist = PCPU_GET(cpulist);
564 if (PCPU_GET(cpumask) & (1 << cpu))
567 cpulist[0] = (uint16_t)cpu;
570 cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
575 ipi_all_but_self(u_int ipi)
577 ipi_selected(PCPU_GET(other_cpus), ipi);