sys/ia64/ia64/mp_machdep.c

   1 /*-
   2  * Copyright (c) 2001-2005 Marcel Moolenaar
   3  * Copyright (c) 2000 Doug Rabson
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_kstack_pages.h"
  32 #include "opt_xtrace.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/ktr.h>
  37 #include <sys/proc.h>
  38 #include <sys/bus.h>
  39 #include <sys/kthread.h>
  40 #include <sys/lock.h>
  41 #include <sys/malloc.h>
  42 #include <sys/mutex.h>
  43 #include <sys/kernel.h>
  44 #include <sys/pcpu.h>
  45 #include <sys/sched.h>
  46 #include <sys/smp.h>
  47 #include <sys/sysctl.h>
  48 #include <sys/uuid.h>
  49
  50 #include <machine/atomic.h>
  51 #include <machine/bootinfo.h>
  52 #include <machine/cpu.h>
  53 #include <machine/fpu.h>
  54 #include <machine/intr.h>
  55 #include <machine/mca.h>
  56 #include <machine/md_var.h>
  57 #include <machine/pal.h>
  58 #include <machine/pcb.h>
  59 #include <machine/sal.h>
  60 #include <machine/smp.h>
  61
  62 #include <vm/vm.h>
  63 #include <vm/pmap.h>
  64 #include <vm/vm_extern.h>
  65 #include <vm/vm_kern.h>
  66
  67 extern uint64_t bdata[];
  68
  69 extern int smp_disabled;
  70
  71 MALLOC_DEFINE(M_SMP, "SMP", "SMP related allocations");
  72
  73 void ia64_ap_startup(void);
  74
  75 #define SAPIC_ID_GET_ID(x)      ((u_int)((x) >> 8) & 0xff)
  76 #define SAPIC_ID_GET_EID(x)     ((u_int)(x) & 0xff)
  77 #define SAPIC_ID_SET(id, eid)   ((u_int)(((id) & 0xff) << 8) | ((eid) & 0xff))
  78
  79 /* State used to wake and bootstrap APs. */
  80 struct ia64_ap_state ia64_ap_state;
  81
  82 int ia64_ipi_ast;
  83 int ia64_ipi_hardclock;
  84 int ia64_ipi_highfp;
  85 int ia64_ipi_nmi;
  86 int ia64_ipi_preempt;
  87 int ia64_ipi_rndzvs;
  88 int ia64_ipi_stop;
  89
  90 static u_int
  91 sz2shft(uint64_t sz)
  92 {
  93         uint64_t s;
  94         u_int shft;
  95
  96         shft = 12;      /* Start with 4K */
  97         s = 1 << shft;
  98         while (s < sz) {
  99                 shft++;
 100                 s <<= 1;
 101         }
 102         return (shft);
 103 }
 104
 105 static u_int
 106 ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf)
 107 {
 108
 109         PCPU_INC(md.stats.pcs_nasts);
 110         CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
 111         return (0);
 112 }
 113
 114 static u_int
 115 ia64_ih_hardclock(struct thread *td, u_int xiv, struct trapframe *tf)
 116 {
 117         struct trapframe *stf;
 118
 119         PCPU_INC(md.stats.pcs_nhardclocks);
 120         CTR1(KTR_SMP, "IPI_HARDCLOCK, cpuid=%d", PCPU_GET(cpuid));
 121         stf = td->td_intr_frame;
 122         td->td_intr_frame = tf;
 123         hardclockintr();
 124         td->td_intr_frame = stf;
 125         return (0);
 126 }
 127
 128 static u_int
 129 ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf)
 130 {
 131
 132         PCPU_INC(md.stats.pcs_nhighfps);
 133         ia64_highfp_save_ipi();
 134         return (0);
 135 }
 136
 137 static u_int
 138 ia64_ih_preempt(struct thread *td, u_int xiv, struct trapframe *tf)
 139 {
 140
 141         PCPU_INC(md.stats.pcs_npreempts);
 142         CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid));
 143         sched_preempt(curthread);
 144         return (0);
 145 }
 146
 147 static u_int
 148 ia64_ih_rndzvs(struct thread *td, u_int xiv, struct trapframe *tf)
 149 {
 150
 151         PCPU_INC(md.stats.pcs_nrdvs);
 152         CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
 153         smp_rendezvous_action();
 154         return (0);
 155 }
 156
 157 static u_int
 158 ia64_ih_stop(struct thread *td, u_int xiv, struct trapframe *tf)
 159 {
 160         u_int cpuid;
 161
 162         PCPU_INC(md.stats.pcs_nstops);
 163         cpuid = PCPU_GET(cpuid);
 164
 165         savectx(PCPU_PTR(md.pcb));
 166
 167         CPU_SET_ATOMIC(cpuid, &stopped_cpus);
 168         while (!CPU_ISSET(cpuid, &started_cpus))
 169                 cpu_spinwait();
 170         CPU_CLR_ATOMIC(cpuid, &started_cpus);
 171         CPU_CLR_ATOMIC(cpuid, &stopped_cpus);
 172         return (0);
 173 }
 174
 175 struct cpu_group *
 176 cpu_topo(void)
 177 {
 178
 179         return smp_topo_none();
 180 }
 181
 182 static void
 183 ia64_store_mca_state(void* arg)
 184 {
 185         struct pcpu *pc = arg;
 186         struct thread *td = curthread;
 187
 188         /*
 189          * ia64_mca_save_state() is CPU-sensitive, so bind ourself to our
 190          * target CPU.
 191          */
 192         thread_lock(td);
 193         sched_bind(td, pc->pc_cpuid);
 194         thread_unlock(td);
 195
 196         ia64_mca_init_ap();
 197
 198         /*
 199          * Get and save the CPU specific MCA records. Should we get the
 200          * MCA state for each processor, or just the CMC state?
 201          */
 202         ia64_mca_save_state(SAL_INFO_MCA);
 203         ia64_mca_save_state(SAL_INFO_CMC);
 204
 205         kproc_exit(0);
 206 }
 207
 208 void
 209 ia64_ap_startup(void)
 210 {
 211         uint64_t vhpt;
 212
 213         ia64_ap_state.as_trace = 0x100;
 214
 215         ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
 216         ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (LOG2_ID_PAGE_SIZE << 2));
 217         ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (LOG2_ID_PAGE_SIZE << 2));
 218         ia64_srlz_d();
 219
 220         pcpup = ia64_ap_state.as_pcpu;
 221         ia64_set_k4((intptr_t)pcpup);
 222
 223         ia64_ap_state.as_trace = 0x108;
 224
 225         vhpt = pcpup->pc_md.vhpt;
 226         map_vhpt(vhpt);
 227         ia64_set_pta(vhpt + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
 228         ia64_srlz_i();
 229
 230         ia64_ap_state.as_trace = 0x110;
 231
 232         ia64_ap_state.as_awake = 1;
 233         ia64_ap_state.as_delay = 0;
 234
 235         map_pal_code();
 236         map_gateway_page();
 237
 238         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 239
 240 #ifdef XTRACE
 241         ia64_xtrace_init_ap(ia64_ap_state.as_xtrace_buffer);
 242 #endif
 243
 244         /* Wait until it's time for us to be unleashed */
 245         while (ia64_ap_state.as_spin)
 246                 cpu_spinwait();
 247
 248         /* Initialize curthread. */
 249         KASSERT(pcpup->pc_idlethread != NULL, ("no idle thread"));
 250         pcpup->pc_curthread = pcpup->pc_idlethread;
 251
 252         pmap_invalidate_all();
 253
 254         atomic_add_int(&ia64_ap_state.as_awake, 1);
 255         while (!smp_started)
 256                 cpu_spinwait();
 257
 258         CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
 259
 260         cpu_initclocks();
 261
 262         ia64_set_tpr(0);
 263         ia64_srlz_d();
 264
 265         sched_throw(NULL);
 266         /* NOTREACHED */
 267 }
 268
 269 void
 270 cpu_mp_setmaxid(void)
 271 {
 272
 273         /*
 274          * Count the number of processors in the system by walking the ACPI
 275          * tables. Note that we record the actual number of processors, even
 276          * if this is larger than MAXCPU. We only activate MAXCPU processors.
 277          */
 278         mp_ncpus = ia64_count_cpus();
 279
 280         /*
 281          * Set the largest cpuid we're going to use. This is necessary for
 282          * VM initialization.
 283          */
 284         mp_maxid = min(mp_ncpus, MAXCPU) - 1;
 285 }
 286
 287 int
 288 cpu_mp_probe(void)
 289 {
 290
 291         /*
 292          * If there's only 1 processor, or we don't have a wake-up vector,
 293          * we're not going to enable SMP. Note that no wake-up vector can
 294          * also mean that the wake-up mechanism is not supported. In this
 295          * case we can have multiple processors, but we simply can't wake
 296          * them up...
 297          */
 298         return (mp_ncpus > 1 && ia64_ipi_wakeup != 0);
 299 }
 300
 301 void
 302 cpu_mp_add(u_int acpi_id, u_int id, u_int eid)
 303 {
 304         struct pcpu *pc;
 305         void *dpcpu;
 306         u_int cpuid, sapic_id;
 307
 308         if (smp_disabled)
 309                 return;
 310
 311         sapic_id = SAPIC_ID_SET(id, eid);
 312         cpuid = (IA64_LID_GET_SAPIC_ID(ia64_get_lid()) == sapic_id)
 313             ? 0 : smp_cpus++;
 314
 315         KASSERT(!CPU_ISSET(cpuid, &all_cpus),
 316             ("%s: cpu%d already in CPU map", __func__, acpi_id));
 317
 318         if (cpuid != 0) {
 319                 pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK);
 320                 pcpu_init(pc, cpuid, sizeof(*pc));
 321                 dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
 322                     M_WAITOK | M_ZERO);
 323                 dpcpu_init(dpcpu, cpuid);
 324         } else
 325                 pc = pcpup;
 326
 327         cpu_pcpu_setup(pc, acpi_id, sapic_id);
 328
 329         CPU_SET(pc->pc_cpuid, &all_cpus);
 330 }
 331
 332 void
 333 cpu_mp_announce()
 334 {
 335         struct pcpu *pc;
 336         uint32_t sapic_id;
 337         int i;
 338
 339         for (i = 0; i <= mp_maxid; i++) {
 340                 pc = pcpu_find(i);
 341                 if (pc != NULL) {
 342                         sapic_id = IA64_LID_GET_SAPIC_ID(pc->pc_md.lid);
 343                         printf("cpu%d: ACPI Id=%x, SAPIC Id=%x, SAPIC Eid=%x",
 344                             i, pc->pc_acpi_id, SAPIC_ID_GET_ID(sapic_id),
 345                             SAPIC_ID_GET_EID(sapic_id));
 346                         if (i == 0)
 347                                 printf(" (BSP)\n");
 348                         else
 349                                 printf("\n");
 350                 }
 351         }
 352 }
 353
 354 void
 355 cpu_mp_start()
 356 {
 357         struct ia64_sal_result result;
 358         struct ia64_fdesc *fd;
 359         struct pcpu *pc;
 360         uintptr_t state;
 361         u_char *stp;
 362
 363         state = ia64_tpa((uintptr_t)&ia64_ap_state);
 364         fd = (struct ia64_fdesc *) os_boot_rendez;
 365         result = ia64_sal_entry(SAL_SET_VECTORS, SAL_OS_BOOT_RENDEZ,
 366             ia64_tpa(fd->func), state, 0, 0, 0, 0);
 367
 368         ia64_ap_state.as_pgtbl_pte = PTE_PRESENT | PTE_MA_WB |
 369             PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW |
 370             (bootinfo->bi_pbvm_pgtbl & PTE_PPN_MASK);
 371         ia64_ap_state.as_pgtbl_itir = sz2shft(bootinfo->bi_pbvm_pgtblsz) << 2;
 372         ia64_ap_state.as_text_va = IA64_PBVM_BASE;
 373         ia64_ap_state.as_text_pte = PTE_PRESENT | PTE_MA_WB |
 374             PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RX |
 375             (ia64_tpa(IA64_PBVM_BASE) & PTE_PPN_MASK);
 376         ia64_ap_state.as_text_itir = bootinfo->bi_text_mapped << 2;
 377         ia64_ap_state.as_data_va = (uintptr_t)bdata;
 378         ia64_ap_state.as_data_pte = PTE_PRESENT | PTE_MA_WB |
 379             PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW |
 380             (ia64_tpa((uintptr_t)bdata) & PTE_PPN_MASK);
 381         ia64_ap_state.as_data_itir = bootinfo->bi_data_mapped << 2;
 382
 383         /* Keep 'em spinning until we unleash them... */
 384         ia64_ap_state.as_spin = 1;
 385
 386         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
 387                 pc->pc_md.current_pmap = kernel_pmap;
 388                 /* The BSP is obviously running already. */
 389                 if (pc->pc_cpuid == 0) {
 390                         pc->pc_md.awake = 1;
 391                         continue;
 392                 }
 393
 394                 ia64_ap_state.as_pcpu = pc;
 395                 pc->pc_md.vhpt = pmap_alloc_vhpt();
 396                 if (pc->pc_md.vhpt == 0) {
 397                         printf("SMP: WARNING: unable to allocate VHPT"
 398                             " for cpu%d", pc->pc_cpuid);
 399                         continue;
 400                 }
 401
 402                 stp = malloc(KSTACK_PAGES * PAGE_SIZE, M_SMP, M_WAITOK);
 403                 ia64_ap_state.as_kstack = stp;
 404                 ia64_ap_state.as_kstack_top = stp + KSTACK_PAGES * PAGE_SIZE;
 405
 406 #ifdef XTRACE
 407                 ia64_ap_state.as_xtrace_buffer = ia64_xtrace_alloc();
 408 #endif
 409
 410                 ia64_ap_state.as_trace = 0;
 411                 ia64_ap_state.as_delay = 2000;
 412                 ia64_ap_state.as_awake = 0;
 413
 414                 if (bootverbose)
 415                         printf("SMP: waking up cpu%d\n", pc->pc_cpuid);
 416
 417                 /* Here she goes... */
 418                 ipi_send(pc, ia64_ipi_wakeup);
 419                 do {
 420                         DELAY(1000);
 421                 } while (--ia64_ap_state.as_delay > 0);
 422
 423                 pc->pc_md.awake = ia64_ap_state.as_awake;
 424
 425                 if (!ia64_ap_state.as_awake) {
 426                         printf("SMP: WARNING: cpu%d did not wake up (code "
 427                             "%#lx)\n", pc->pc_cpuid,
 428                             ia64_ap_state.as_trace - state);
 429                 }
 430         }
 431 }
 432
 433 static void
 434 cpu_mp_unleash(void *dummy)
 435 {
 436         struct pcpu *pc;
 437         int cpus;
 438
 439         if (mp_ncpus <= 1)
 440                 return;
 441
 442         /* Allocate XIVs for IPIs */
 443         ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast);
 444         ia64_ipi_hardclock = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI,
 445             ia64_ih_hardclock);
 446         ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp);
 447         ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI,
 448             ia64_ih_preempt);
 449         ia64_ipi_rndzvs = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_rndzvs);
 450         ia64_ipi_stop = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI, ia64_ih_stop);
 451
 452         /* Reserve the NMI vector for IPI_STOP_HARD if possible */
 453         ia64_ipi_nmi = (ia64_xiv_reserve(2, IA64_XIV_IPI, ia64_ih_stop) != 0)
 454             ? ia64_ipi_stop : 0x400;    /* DM=NMI, Vector=n/a */
 455
 456         cpus = 0;
 457         smp_cpus = 0;
 458         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
 459                 cpus++;
 460                 if (pc->pc_md.awake) {
 461                         kproc_create(ia64_store_mca_state, pc, NULL, 0, 0,
 462                             "mca %u", pc->pc_cpuid);
 463                         smp_cpus++;
 464                 }
 465         }
 466
 467         ia64_ap_state.as_awake = 1;
 468         ia64_ap_state.as_spin = 0;
 469
 470         while (ia64_ap_state.as_awake != smp_cpus)
 471                 cpu_spinwait();
 472
 473         if (smp_cpus != cpus || cpus != mp_ncpus) {
 474                 printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
 475                     mp_ncpus, cpus, smp_cpus);
 476         }
 477
 478         /* XXX Atomic set operation? */
 479         smp_started = 1;
 480
 481         /*
 482          * Now that all CPUs are up and running, bind interrupts to each of
 483          * them.
 484          */
 485         ia64_bind_intr();
 486 }
 487 SYSINIT(start_aps, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, cpu_mp_unleash, NULL);
 488
 489 /*
 490  * send an IPI to a set of cpus.
 491  */
 492 void
 493 ipi_selected(cpuset_t cpus, int ipi)
 494 {
 495         struct pcpu *pc;
 496
 497         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
 498                 if (CPU_ISSET(pc->pc_cpuid, &cpus))
 499                         ipi_send(pc, ipi);
 500         }
 501 }
 502
 503 /*
 504  * send an IPI to a specific CPU.
 505  */
 506 void
 507 ipi_cpu(int cpu, u_int ipi)
 508 {
 509
 510         ipi_send(cpuid_to_pcpu[cpu], ipi);
 511 }
 512
 513 /*
 514  * send an IPI to all CPUs EXCEPT myself.
 515  */
 516 void
 517 ipi_all_but_self(int ipi)
 518 {
 519         struct pcpu *pc;
 520
 521         STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
 522                 if (pc != pcpup)
 523                         ipi_send(pc, ipi);
 524         }
 525 }
 526
 527 /*
 528  * Send an IPI to the specified processor.
 529  */
 530 void
 531 ipi_send(struct pcpu *cpu, int xiv)
 532 {
 533         u_int sapic_id;
 534
 535         KASSERT(xiv != 0, ("ipi_send"));
 536
 537         sapic_id = IA64_LID_GET_SAPIC_ID(cpu->pc_md.lid);
 538
 539         ia64_mf();
 540         ia64_st8(&(ia64_pib->ib_ipi[sapic_id][0]), xiv);
 541         ia64_mf_a();
 542         CTR3(KTR_SMP, "ipi_send(%p, %d): cpuid=%d", cpu, xiv, PCPU_GET(cpuid));
 543 }