2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright 2019 Justin Hibbits
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
31 #include "opt_platform.h"
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/module.h>
38 #include <sys/endian.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
46 #include <machine/bus.h>
47 #include <machine/intr_machdep.h>
48 #include <machine/md_var.h>
50 #include <dev/ofw/ofw_bus.h>
51 #include <dev/ofw/ofw_bus_subr.h>
54 #include <powerpc/powernv/opal.h>
59 #define XIVE_PRIORITY 7 /* Random non-zero number */
60 #define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */
63 #define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */
64 #define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */
65 #define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */
67 #define XIVE_TM_NSR 0x00
68 #define XIVE_TM_CPPR 0x01
69 #define XIVE_TM_IPB 0x02
70 #define XIVE_TM_LSMFB 0x03
71 #define XIVE_TM_ACK_CNT 0x04
72 #define XIVE_TM_INC 0x05
73 #define XIVE_TM_AGE 0x06
74 #define XIVE_TM_PIPR 0x07
78 #define TM_QW2W2_VP 0x80000000
80 #define XIVE_TM_SPC_ACK 0x800
81 #define TM_QW3NSR_HE_SHIFT 14
82 #define TM_QW3_NSR_HE_NONE 0
83 #define TM_QW3_NSR_HE_POOL 1
84 #define TM_QW3_NSR_HE_PHYS 2
85 #define TM_QW3_NSR_HE_LSI 3
86 #define XIVE_TM_SPC_PULL_POOL_CTX 0x828
88 #define XIVE_IRQ_LOAD_EOI 0x000
89 #define XIVE_IRQ_STORE_EOI 0x400
90 #define XIVE_IRQ_PQ_00 0xc00
91 #define XIVE_IRQ_PQ_01 0xd00
93 #define XIVE_IRQ_VAL_P 0x02
94 #define XIVE_IRQ_VAL_Q 0x01
99 extern void (*powernv_smp_ap_extra_init)(void);
101 /* Private support */
102 static void xive_setup_cpu(void);
103 static void xive_smp_cpu_startup(void);
104 static void xive_init_irq(struct xive_irq *irqd, u_int irq);
105 static struct xive_irq *xive_configure_irq(u_int irq);
106 static int xive_provision_page(struct xive_softc *sc);
110 static int xive_probe(device_t);
111 static int xive_attach(device_t);
112 static int xics_probe(device_t);
113 static int xics_attach(device_t);
115 static void xive_bind(device_t, u_int, cpuset_t, void **);
116 static void xive_dispatch(device_t, struct trapframe *);
117 static void xive_enable(device_t, u_int, u_int, void **);
118 static void xive_eoi(device_t, u_int, void *);
119 static void xive_ipi(device_t, u_int);
120 static void xive_mask(device_t, u_int, void *);
121 static void xive_unmask(device_t, u_int, void *);
122 static void xive_translate_code(device_t dev, u_int irq, int code,
123 enum intr_trigger *trig, enum intr_polarity *pol);
125 static device_method_t xive_methods[] = {
126 /* Device interface */
127 DEVMETHOD(device_probe, xive_probe),
128 DEVMETHOD(device_attach, xive_attach),
131 DEVMETHOD(pic_bind, xive_bind),
132 DEVMETHOD(pic_dispatch, xive_dispatch),
133 DEVMETHOD(pic_enable, xive_enable),
134 DEVMETHOD(pic_eoi, xive_eoi),
135 DEVMETHOD(pic_ipi, xive_ipi),
136 DEVMETHOD(pic_mask, xive_mask),
137 DEVMETHOD(pic_unmask, xive_unmask),
138 DEVMETHOD(pic_translate_code, xive_translate_code),
143 static device_method_t xics_methods[] = {
144 /* Device interface */
145 DEVMETHOD(device_probe, xics_probe),
146 DEVMETHOD(device_attach, xics_attach),
153 struct resource *sc_mem;
154 vm_size_t sc_prov_page_size;
160 uint32_t *q_eoi_page;
172 #define OPAL_XIVE_IRQ_EOI_VIA_FW 0x00000020
173 #define OPAL_XIVE_IRQ_MASK_VIA_FW 0x00000010
174 #define OPAL_XIVE_IRQ_SHIFT_BUG 0x00000008
175 #define OPAL_XIVE_IRQ_LSI 0x00000004
176 #define OPAL_XIVE_IRQ_STORE_EOI 0x00000002
177 #define OPAL_XIVE_IRQ_TRIGGER_PAGE 0x00000001
179 vm_offset_t eoi_page;
180 vm_offset_t trig_page;
188 struct xive_irq ipi_data;
189 struct xive_queue queue; /* We only use a single queue for now. */
194 static driver_t xive_driver = {
197 sizeof(struct xive_softc)
200 static driver_t xics_driver = {
206 static devclass_t xive_devclass;
207 static devclass_t xics_devclass;
209 EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, xive_devclass, 0, 0,
210 BUS_PASS_INTERRUPT-1);
211 EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, xics_devclass, 0, 0,
214 MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
216 DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data);
218 static int xive_ipi_vector = -1;
221 * XIVE Exploitation mode driver.
223 * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
224 * mode, and "Exploitation mode". XICS emulation mode is compatible with the
225 * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate
226 * hypervisor calls and memory accesses. Exploitation mode gives us raw access
227 * to the XIVE MMIO, improving performance significantly.
229 * The XIVE controller is a very bizarre interrupt controller. It uses queues
230 * in memory to pass interrupts around, and maps itself into 512GB of physical
231 * device address space, giving each interrupt in the system one or more pages
232 * of address space. An IRQ is tied to a virtual processor, which could be a
233 * physical CPU thread, or a guest CPU thread (LPAR running on a physical
234 * thread). Thus, the controller can route interrupts directly to guest OSes
235 * bypassing processing by the hypervisor, thereby improving performance of the
238 * An IRQ, in addition to being tied to a virtual processor, has one or two
239 * page mappings: an EOI page, and an optional trigger page. The trigger page
240 * could be the same as the EOI page. Level-sensitive interrupts (LSIs) don't
241 * have a trigger page, as they're external interrupts controlled by physical
242 * lines. MSIs and IPIs have trigger pages. An IPI is really just another IRQ
243 * in the XIVE, which is triggered by software.
245 * An interesting behavior of the XIVE controller is that oftentimes the
246 * contents of an address location don't actually matter, but the direction of
247 * the action is the signifier (read vs write), and the address is significant.
248 * Hence, masking and unmasking an interrupt is done by reading different
249 * addresses in the EOI page, and triggering an interrupt consists of writing to
252 * Additionally, the MMIO region mapped is CPU-sensitive, just like the
253 * per-processor register space (private access) in OpenPIC. In order for a CPU
254 * to receive interrupts it must itself configure its CPPR (Current Processor
255 * Priority Register), it cannot be set by any other processor. This
256 * necessitates the xive_smp_cpu_startup() function.
258 * Queues are pages of memory, sized powers-of-two, that are shared with the
259 * XIVE. The XIVE writes into the queue with an alternating polarity bit, which
260 * flips when the queue wraps.
264 * Offset-based read/write interfaces.
267 xive_read_2(struct xive_softc *sc, bus_size_t offset)
270 return (bus_read_2(sc->sc_mem, sc->sc_offset + offset));
274 xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val)
277 bus_write_1(sc->sc_mem, sc->sc_offset + offset, val);
280 /* EOI and Trigger page access interfaces. */
282 xive_read_mmap8(vm_offset_t addr)
284 return (*(volatile uint64_t *)addr);
288 xive_write_mmap8(vm_offset_t addr, uint64_t val)
290 *(uint64_t *)(addr) = val;
294 /* Device interfaces. */
296 xive_probe(device_t dev)
299 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
302 device_set_desc(dev, "External Interrupt Virtualization Engine");
304 /* Make sure we always win against the xicp driver. */
305 return (BUS_PROBE_DEFAULT);
309 xics_probe(device_t dev)
312 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
315 device_set_desc(dev, "External Interrupt Virtualization Engine Root");
316 return (BUS_PROBE_DEFAULT);
320 xive_attach(device_t dev)
322 struct xive_softc *sc = device_get_softc(dev);
323 struct xive_cpu *xive_cpud;
324 phandle_t phandle = ofw_bus_get_node(dev);
332 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
334 error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
335 (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
337 rid = 1; /* Get the Hypervisor-level register set. */
338 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
340 sc->sc_offset = XIVE_TM_QW3_HV;
342 mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
344 order = fls(mp_maxid + (mp_maxid - 1)) - 1;
347 vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
348 if (vp_block == OPAL_BUSY)
350 else if (vp_block == OPAL_XIVE_PROVISIONING)
351 xive_provision_page(sc);
358 "Unable to allocate VP block. Opal error %d\n",
360 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
365 * Set up the VPs. Try to do as much as we can in attach, to lessen
366 * what's needed at AP spawn time.
369 vp_id = pcpu_find(i)->pc_hwref;
371 xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
372 xive_cpud->vp = vp_id + vp_block;
373 opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
374 vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
376 /* Allocate the queue page and populate the queue state data. */
377 xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE,
378 M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
379 xive_cpud->queue.q_size = 1 << PAGE_SHIFT;
380 xive_cpud->queue.q_mask =
381 ((xive_cpud->queue.q_size / sizeof(int)) - 1);
382 xive_cpud->queue.q_toggle = 0;
383 xive_cpud->queue.q_index = 0;
385 error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
386 OPAL_XIVE_VP_ENABLED, 0);
387 } while (error == OPAL_BUSY);
388 error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id,
389 XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT,
390 OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
393 ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ,
395 } while (ipi_irq == OPAL_BUSY);
398 device_printf(root_pic,
399 "Failed allocating IPI. OPAL error %d\n",
402 xive_init_irq(&xive_cpud->ipi_data, ipi_irq);
403 xive_cpud->ipi_data.vp = vp_id;
404 xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
405 opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq,
406 xive_cpud->ipi_data.vp, XIVE_PRIORITY,
411 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
412 1 /* Number of IPIs */, FALSE);
416 powernv_smp_ap_extra_init = xive_smp_cpu_startup;
422 xics_attach(device_t dev)
424 phandle_t phandle = ofw_bus_get_node(dev);
426 /* The XIVE (root PIC) will handle all our interrupts */
427 powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
428 MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
438 xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
440 struct xive_irq *irqd;
445 *priv = xive_configure_irq(irq);
450 * This doesn't appear to actually support affinity groups, so pick a
455 if (CPU_ISSET(cpu, &cpumask)) ncpus++;
460 if (!CPU_ISSET(cpu, &cpumask))
467 opal_call(OPAL_XIVE_SYNC);
469 irqd->vp = pcpu_find(cpu)->pc_hwref;
470 error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp,
471 XIVE_PRIORITY, irqd->lirq);
474 panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
476 xive_eoi(dev, irq, irqd);
479 /* Read the next entry in the queue page and update the index. */
481 xive_read_eq(struct xive_queue *q)
483 uint32_t i = be32toh(q->q_page[q->q_index]);
485 /* Check validity, using current queue polarity. */
486 if ((i >> 31) == q->q_toggle)
489 q->q_index = (q->q_index + 1) & q->q_mask;
494 return (i & 0x7fffffff);
498 xive_dispatch(device_t dev, struct trapframe *tf)
500 struct xive_softc *sc;
501 struct xive_cpu *xive_cpud;
506 sc = device_get_softc(dev);
509 ack = xive_read_2(sc, XIVE_TM_SPC_ACK);
512 he = ack >> TM_QW3NSR_HE_SHIFT;
514 if (he == TM_QW3_NSR_HE_NONE)
517 case TM_QW3_NSR_HE_NONE:
519 case TM_QW3_NSR_HE_POOL:
520 case TM_QW3_NSR_HE_LSI:
522 "Unexpected interrupt he type: %d\n", he);
524 case TM_QW3_NSR_HE_PHYS:
528 xive_cpud = DPCPU_PTR(xive_cpu_data);
529 xive_write_1(sc, XIVE_TM_CPPR, cppr);
532 vector = xive_read_eq(&xive_cpud->queue);
537 if (vector == MAX_XIVE_IRQS)
538 vector = xive_ipi_vector;
540 powerpc_dispatch_intr(vector, tf);
544 xive_write_1(sc, XIVE_TM_CPPR, 0xff);
548 xive_enable(device_t dev, u_int irq, u_int vector, void **priv)
550 struct xive_irq *irqd;
553 if (irq == MAX_XIVE_IRQS) {
554 if (xive_ipi_vector == -1)
555 xive_ipi_vector = vector;
559 *priv = xive_configure_irq(irq);
563 /* Bind to this CPU to start */
564 cpu = PCPU_GET(hwref);
568 status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu,
569 XIVE_PRIORITY, vector);
570 if (status != OPAL_BUSY)
576 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
579 xive_unmask(dev, irq, *priv);
583 xive_eoi(device_t dev, u_int irq, void *priv)
585 struct xive_irq *rirq;
586 struct xive_cpu *cpud;
589 if (irq == MAX_XIVE_IRQS) {
590 cpud = DPCPU_PTR(xive_cpu_data);
591 rirq = &cpud->ipi_data;
595 if (rirq->flags & OPAL_XIVE_IRQ_EOI_VIA_FW)
596 opal_call(OPAL_INT_EOI, irq);
597 else if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI)
598 xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0);
599 else if (rirq->flags & OPAL_XIVE_IRQ_LSI)
600 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI);
602 eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
603 if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0)
604 xive_write_mmap8(rirq->trig_page, 0);
609 xive_ipi(device_t dev, u_int cpu)
611 struct xive_cpu *xive_cpud;
613 xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data);
615 if (xive_cpud->ipi_data.trig_page == 0)
617 xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0);
621 xive_mask(device_t dev, u_int irq, void *priv)
623 struct xive_irq *rirq;
625 /* Never mask IPIs */
626 if (irq == MAX_XIVE_IRQS)
631 if (!(rirq->flags & OPAL_XIVE_IRQ_LSI))
633 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01);
637 xive_unmask(device_t dev, u_int irq, void *priv)
639 struct xive_irq *rirq;
643 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
647 xive_translate_code(device_t dev, u_int irq, int code,
648 enum intr_trigger *trig, enum intr_polarity *pol)
653 *trig = INTR_TRIGGER_EDGE;
654 *pol = INTR_POLARITY_HIGH;
658 *trig = INTR_TRIGGER_LEVEL;
659 *pol = INTR_POLARITY_LOW;
662 *trig = INTR_TRIGGER_CONFORM;
663 *pol = INTR_POLARITY_CONFORM;
667 /* Private functions. */
669 * Setup the current CPU. Called by the BSP at driver attachment, and by each
670 * AP at wakeup (via xive_smp_cpu_startup()).
675 struct xive_softc *sc;
676 struct xive_cpu *cpup;
679 cpup = DPCPU_PTR(xive_cpu_data);
681 sc = device_get_softc(root_pic);
683 val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
684 if (val & TM_QW2W2_VP)
685 bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
687 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
688 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
689 TM_QW2W2_VP | cpup->cam);
691 xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data);
692 xive_write_1(sc, XIVE_TM_CPPR, 0xff);
695 /* Populate an IRQ structure, mapping the EOI and trigger pages. */
697 xive_init_irq(struct xive_irq *irqd, u_int irq)
699 uint64_t eoi_phys, trig_phys;
702 opal_call(OPAL_XIVE_GET_IRQ_INFO, irq,
703 vtophys(&irqd->flags), vtophys(&eoi_phys),
704 vtophys(&trig_phys), vtophys(&esb_shift),
705 vtophys(&irqd->chip));
708 irqd->esb_size = 1 << esb_shift;
709 irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
711 if (eoi_phys == trig_phys)
712 irqd->trig_page = irqd->eoi_page;
713 else if (trig_phys != 0)
714 irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys,
719 opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp),
720 vtophys(&irqd->prio), vtophys(&irqd->lirq));
723 /* Allocate an IRQ struct before populating it. */
724 static struct xive_irq *
725 xive_configure_irq(u_int irq)
727 struct xive_irq *irqd;
729 irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK);
731 xive_init_irq(irqd, irq);
737 * Part of the OPAL API. OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages,
738 * provisioned through this call.
741 xive_provision_page(struct xive_softc *sc)
747 prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
748 0, BUS_SPACE_MAXADDR,
749 sc->sc_prov_page_size, sc->sc_prov_page_size);
751 error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
753 } while (error == OPAL_XIVE_PROVISIONING);
758 /* The XIVE_TM_CPPR register must be set by each thread */
760 xive_smp_cpu_startup(void)