2 * Copyright (c) 2015 Nathan Whitehorn
3 * Copyright (c) 2017-2018 Semihalf
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
41 #include <machine/bus.h>
42 #include <machine/cpu.h>
43 #include <machine/hid.h>
44 #include <machine/platformvar.h>
45 #include <machine/pmap.h>
46 #include <machine/rtas.h>
47 #include <machine/smp.h>
48 #include <machine/spr.h>
49 #include <machine/trap.h>
51 #include <dev/ofw/openfirm.h>
52 #include <dev/ofw/ofw_bus.h>
53 #include <dev/ofw/ofw_bus_subr.h>
54 #include <machine/ofw_machdep.h>
55 #include <powerpc/aim/mmu_oea64.h>
57 #include "platform_if.h"
64 void (*powernv_smp_ap_extra_init)(void);
66 static int powernv_probe(platform_t);
67 static int powernv_attach(platform_t);
68 void powernv_mem_regions(platform_t, struct mem_region *phys, int *physsz,
69 struct mem_region *avail, int *availsz);
70 static void powernv_numa_mem_regions(platform_t plat, struct numa_mem_region *phys, int *physsz);
71 static u_long powernv_timebase_freq(platform_t, struct cpuref *cpuref);
72 static int powernv_smp_first_cpu(platform_t, struct cpuref *cpuref);
73 static int powernv_smp_next_cpu(platform_t, struct cpuref *cpuref);
74 static int powernv_smp_get_bsp(platform_t, struct cpuref *cpuref);
75 static void powernv_smp_ap_init(platform_t);
77 static int powernv_smp_start_cpu(platform_t, struct pcpu *cpu);
78 static void powernv_smp_probe_threads(platform_t);
79 static struct cpu_group *powernv_smp_topo(platform_t plat);
81 static void powernv_reset(platform_t);
82 static void powernv_cpu_idle(sbintime_t sbt);
83 static int powernv_cpuref_init(void);
84 static int powernv_node_numa_domain(platform_t platform, phandle_t node);
86 static platform_method_t powernv_methods[] = {
87 PLATFORMMETHOD(platform_probe, powernv_probe),
88 PLATFORMMETHOD(platform_attach, powernv_attach),
89 PLATFORMMETHOD(platform_mem_regions, powernv_mem_regions),
90 PLATFORMMETHOD(platform_numa_mem_regions, powernv_numa_mem_regions),
91 PLATFORMMETHOD(platform_timebase_freq, powernv_timebase_freq),
93 PLATFORMMETHOD(platform_smp_ap_init, powernv_smp_ap_init),
94 PLATFORMMETHOD(platform_smp_first_cpu, powernv_smp_first_cpu),
95 PLATFORMMETHOD(platform_smp_next_cpu, powernv_smp_next_cpu),
96 PLATFORMMETHOD(platform_smp_get_bsp, powernv_smp_get_bsp),
98 PLATFORMMETHOD(platform_smp_start_cpu, powernv_smp_start_cpu),
99 PLATFORMMETHOD(platform_smp_probe_threads, powernv_smp_probe_threads),
100 PLATFORMMETHOD(platform_smp_topo, powernv_smp_topo),
102 PLATFORMMETHOD(platform_node_numa_domain, powernv_node_numa_domain),
104 PLATFORMMETHOD(platform_reset, powernv_reset),
108 static platform_def_t powernv_platform = {
114 static struct cpuref platform_cpuref[MAXCPU];
115 static int platform_cpuref_cnt;
116 static int platform_cpuref_valid;
117 static int platform_associativity;
119 PLATFORM_DEF(powernv_platform);
121 static uint64_t powernv_boot_pir;
124 powernv_probe(platform_t plat)
126 if (opal_check() == 0)
127 return (BUS_PROBE_SPECIFIC);
133 powernv_attach(platform_t plat)
135 uint32_t nptlp, shift = 0, slb_encoding = 0;
136 int32_t lp_size, lp_encoding;
138 pcell_t refpoints[3];
146 /* Ping OPAL again just to make sure */
149 #if BYTE_ORDER == LITTLE_ENDIAN
150 opal_call(OPAL_REINIT_CPUS, 2 /* Little endian */);
152 opal_call(OPAL_REINIT_CPUS, 1 /* Big endian */);
154 opal = OF_finddevice("/ibm,opal");
156 platform_associativity = 4; /* Skiboot default. */
157 if (OF_getencprop(opal, "ibm,associativity-reference-points", refpoints,
158 sizeof(refpoints)) > 0) {
159 platform_associativity = refpoints[0];
162 if (cpu_idle_hook == NULL)
163 cpu_idle_hook = powernv_cpu_idle;
165 powernv_boot_pir = mfspr(SPR_PIR);
167 /* LPID must not be altered when PSL_DR or PSL_IR is set */
169 mtmsr(msr & ~(PSL_DR | PSL_IR));
171 /* Direct interrupts to SRR instead of HSRR and reset LPCR otherwise */
175 if (cpu_features2 & PPC_FEATURE2_ARCH_3_00)
178 #if BYTE_ORDER == LITTLE_ENDIAN
182 mtspr(SPR_LPCR, lpcr);
187 powernv_cpuref_init();
189 /* Set SLB count from device tree */
193 res = OF_getprop(cpu, "name", buf, sizeof(buf));
194 if (res > 0 && strcmp(buf, "cpus") == 0)
203 res = OF_getprop(cpu, "device_type", buf, sizeof(buf));
204 if (res > 0 && strcmp(buf, "cpu") == 0)
211 res = OF_getencprop(cpu, "ibm,slb-size", &prop, sizeof(prop));
216 * Scan the large page size property for PAPR compatible machines.
217 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties'
218 * for the encoding of the property.
221 len = OF_getproplen(cpu, "ibm,segment-page-sizes");
224 * We have to use a variable length array on the stack
225 * since we have very limited stack space.
227 pcell_t arr[len/sizeof(cell_t)];
228 res = OF_getencprop(cpu, "ibm,segment-page-sizes", arr,
235 slb_encoding = arr[idx + 1];
236 nptlp = arr[idx + 2];
239 while (len > 0 && nptlp) {
241 lp_encoding = arr[idx+1];
242 if (slb_encoding == SLBV_L && lp_encoding == 0)
245 if (slb_encoding == SLB_PGSZ_4K_4K &&
246 lp_encoding == LP_4K_16M)
247 moea64_has_lp_4k_16m = true;
253 if (has_lp && moea64_has_lp_4k_16m)
258 panic("Standard large pages (SLB[L] = 1, PTE[LP] = 0) "
259 "not supported by this system.");
261 moea64_large_page_shift = shift;
262 moea64_large_page_size = 1ULL << lp_size;
270 powernv_mem_regions(platform_t plat, struct mem_region *phys, int *physsz,
271 struct mem_region *avail, int *availsz)
274 ofw_mem_regions(phys, physsz, avail, availsz);
278 powernv_numa_mem_regions(platform_t plat, struct numa_mem_region *phys, int *physsz)
281 ofw_numa_mem_regions(phys, physsz);
285 powernv_timebase_freq(platform_t plat, struct cpuref *cpuref)
288 phandle_t cpu, dev, root;
293 dev = OF_child(root);
295 res = OF_getprop(dev, "name", buf, sizeof(buf));
296 if (res > 0 && strcmp(buf, "cpus") == 0)
301 for (cpu = OF_child(dev); cpu != 0; cpu = OF_peer(cpu)) {
302 res = OF_getprop(cpu, "device_type", buf, sizeof(buf));
303 if (res > 0 && strcmp(buf, "cpu") == 0)
309 OF_getencprop(cpu, "timebase-frequency", &ticks, sizeof(ticks));
312 panic("Unable to determine timebase frequency!");
319 powernv_cpuref_init(void)
323 int a, res, tmp_cpuref_cnt;
324 static struct cpuref tmp_cpuref[MAXCPU];
325 cell_t interrupt_servers[32];
328 if (platform_cpuref_valid)
334 res = OF_getprop(dev, "name", buf, sizeof(buf));
335 if (res > 0 && strcmp(buf, "cpus") == 0)
342 for (cpu = OF_child(dev); cpu != 0; cpu = OF_peer(cpu)) {
343 res = OF_getprop(cpu, "device_type", buf, sizeof(buf));
344 if (res > 0 && strcmp(buf, "cpu") == 0) {
345 if (!ofw_bus_node_status_okay(cpu))
347 res = OF_getproplen(cpu, "ibm,ppc-interrupt-server#s");
349 OF_getencprop(cpu, "ibm,ppc-interrupt-server#s",
350 interrupt_servers, res);
352 for (a = 0; a < res/sizeof(cell_t); a++) {
353 tmp_cpuref[tmp_cpuref_cnt].cr_hwref = interrupt_servers[a];
354 tmp_cpuref[tmp_cpuref_cnt].cr_cpuid = tmp_cpuref_cnt;
355 tmp_cpuref[tmp_cpuref_cnt].cr_domain =
356 powernv_node_numa_domain(NULL, cpu);
357 if (interrupt_servers[a] == (uint32_t)powernv_boot_pir)
358 bsp = tmp_cpuref_cnt;
366 /* Map IDs, so BSP has CPUID 0 regardless of hwref */
367 for (a = bsp; a < tmp_cpuref_cnt; a++) {
368 platform_cpuref[platform_cpuref_cnt].cr_hwref = tmp_cpuref[a].cr_hwref;
369 platform_cpuref[platform_cpuref_cnt].cr_cpuid = platform_cpuref_cnt;
370 platform_cpuref[platform_cpuref_cnt].cr_domain = tmp_cpuref[a].cr_domain;
371 platform_cpuref_cnt++;
373 for (a = 0; a < bsp; a++) {
374 platform_cpuref[platform_cpuref_cnt].cr_hwref = tmp_cpuref[a].cr_hwref;
375 platform_cpuref[platform_cpuref_cnt].cr_cpuid = platform_cpuref_cnt;
376 platform_cpuref[platform_cpuref_cnt].cr_domain = tmp_cpuref[a].cr_domain;
377 platform_cpuref_cnt++;
380 platform_cpuref_valid = 1;
386 powernv_smp_first_cpu(platform_t plat, struct cpuref *cpuref)
388 if (platform_cpuref_valid == 0)
391 cpuref->cr_cpuid = 0;
392 cpuref->cr_hwref = platform_cpuref[0].cr_hwref;
393 cpuref->cr_domain = platform_cpuref[0].cr_domain;
399 powernv_smp_next_cpu(platform_t plat, struct cpuref *cpuref)
403 if (platform_cpuref_valid == 0)
406 id = cpuref->cr_cpuid + 1;
407 if (id >= platform_cpuref_cnt)
410 cpuref->cr_cpuid = platform_cpuref[id].cr_cpuid;
411 cpuref->cr_hwref = platform_cpuref[id].cr_hwref;
412 cpuref->cr_domain = platform_cpuref[id].cr_domain;
418 powernv_smp_get_bsp(platform_t plat, struct cpuref *cpuref)
421 cpuref->cr_cpuid = platform_cpuref[0].cr_cpuid;
422 cpuref->cr_hwref = platform_cpuref[0].cr_hwref;
423 cpuref->cr_domain = platform_cpuref[0].cr_domain;
429 powernv_smp_start_cpu(platform_t plat, struct pcpu *pc)
436 result = opal_call(OPAL_START_CPU, pc->pc_hwref, EXC_RST);
437 if (result != OPAL_SUCCESS) {
438 printf("OPAL error (%d): unable to start AP %d\n",
439 result, (int)pc->pc_hwref);
447 powernv_smp_probe_threads(platform_t plat)
450 phandle_t cpu, dev, root;
455 dev = OF_child(root);
457 res = OF_getprop(dev, "name", buf, sizeof(buf));
458 if (res > 0 && strcmp(buf, "cpus") == 0)
464 for (cpu = OF_child(dev); cpu != 0; cpu = OF_peer(cpu)) {
465 res = OF_getprop(cpu, "device_type", buf, sizeof(buf));
466 if (res <= 0 || strcmp(buf, "cpu") != 0)
469 res = OF_getproplen(cpu, "ibm,ppc-interrupt-server#s");
472 nthreads = res / sizeof(cell_t);
478 smp_threads_per_core = nthreads;
479 if (mp_ncpus % nthreads == 0)
480 mp_ncores = mp_ncpus / nthreads;
483 static struct cpu_group *
484 powernv_smp_topo(platform_t plat)
486 if (mp_ncpus % smp_threads_per_core != 0) {
487 printf("WARNING: Irregular SMP topology. Performance may be "
488 "suboptimal (%d threads, %d on first core)\n",
489 mp_ncpus, smp_threads_per_core);
490 return (smp_topo_none());
493 /* Don't do anything fancier for non-threaded SMP */
494 if (smp_threads_per_core == 1)
495 return (smp_topo_none());
497 return (smp_topo_1level(CG_SHARE_L1, smp_threads_per_core,
504 powernv_reset(platform_t platform)
507 opal_call(OPAL_CEC_REBOOT);
511 powernv_smp_ap_init(platform_t platform)
514 if (powernv_smp_ap_extra_init != NULL)
515 powernv_smp_ap_extra_init();
519 powernv_cpu_idle(sbintime_t sbt)
524 powernv_node_numa_domain(platform_t platform, phandle_t node)
526 /* XXX: Is locking necessary in here? */
527 static int numa_domains[MAXMEMDOM];
528 static int numa_max_domain;
529 cell_t associativity[5];
535 if (vm_ndomains == 1)
538 res = OF_getencprop(node, "ibm,associativity",
539 associativity, sizeof(associativity));
542 * If this node doesn't have associativity, or if there are not
543 * enough elements in it, check its parent.
545 if (res < (int)(sizeof(cell_t) * (platform_associativity + 1))) {
546 node = OF_parent(node);
547 /* If already at the root, use default domain. */
550 return (powernv_node_numa_domain(platform, node));
553 for (i = 0; i < numa_max_domain; i++) {
554 if (numa_domains[i] == associativity[platform_associativity])
558 numa_domains[numa_max_domain++] =
559 associativity[platform_associativity];
566 /* Set up the Nest MMU on POWER9 relatively early, but after pmap is setup. */
568 powernv_setup_nmmu(void *unused)
570 if (opal_check() != 0)
572 opal_call(OPAL_NMMU_SET_PTCR, -1, mfspr(SPR_PTCR));
575 SYSINIT(powernv_setup_nmmu, SI_SUB_CPU, SI_ORDER_ANY, powernv_setup_nmmu, NULL);