2 * Copyright (c) 2015-2016 The FreeBSD Foundation
4 * This software was developed by Andrew Turner under
5 * the sponsorship of the FreeBSD Foundation.
7 * This software was developed by Semihalf under
8 * the sponsorship of the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include "opt_platform.h"
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/bitstring.h>
42 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
49 #include <sys/cpuset.h>
51 #include <sys/mutex.h>
53 #include <sys/interrupt.h>
58 #include <machine/bus.h>
59 #include <machine/cpu.h>
60 #include <machine/intr.h>
63 #include <dev/fdt/fdt_intr.h>
64 #include <dev/ofw/ofw_bus_subr.h>
68 #include <contrib/dev/acpica/include/acpi.h>
69 #include <dev/acpica/acpivar.h>
76 #include <arm/arm/gic_common.h>
77 #include "gic_v3_reg.h"
78 #include "gic_v3_var.h"
80 static bus_get_domain_t gic_v3_get_domain;
81 static bus_read_ivar_t gic_v3_read_ivar;
82 static bus_write_ivar_t gic_v3_write_ivar;
84 static pic_disable_intr_t gic_v3_disable_intr;
85 static pic_enable_intr_t gic_v3_enable_intr;
86 static pic_map_intr_t gic_v3_map_intr;
87 static pic_setup_intr_t gic_v3_setup_intr;
88 static pic_teardown_intr_t gic_v3_teardown_intr;
89 static pic_post_filter_t gic_v3_post_filter;
90 static pic_post_ithread_t gic_v3_post_ithread;
91 static pic_pre_ithread_t gic_v3_pre_ithread;
92 static pic_bind_intr_t gic_v3_bind_intr;
94 static pic_init_secondary_t gic_v3_init_secondary;
95 static pic_ipi_send_t gic_v3_ipi_send;
96 static pic_ipi_setup_t gic_v3_ipi_setup;
99 static gic_reserve_msi_range_t gic_v3_reserve_msi_range;
100 static gic_alloc_msi_t gic_v3_gic_alloc_msi;
101 static gic_release_msi_t gic_v3_gic_release_msi;
102 static gic_alloc_msix_t gic_v3_gic_alloc_msix;
103 static gic_release_msix_t gic_v3_gic_release_msix;
105 static msi_alloc_msi_t gic_v3_alloc_msi;
106 static msi_release_msi_t gic_v3_release_msi;
107 static msi_alloc_msix_t gic_v3_alloc_msix;
108 static msi_release_msix_t gic_v3_release_msix;
109 static msi_map_msi_t gic_v3_map_msi;
111 static u_int gic_irq_cpu;
113 static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1];
114 static u_int sgi_first_unused = GIC_FIRST_SGI;
117 static device_method_t gic_v3_methods[] = {
118 /* Device interface */
119 DEVMETHOD(device_detach, gic_v3_detach),
122 DEVMETHOD(bus_get_domain, gic_v3_get_domain),
123 DEVMETHOD(bus_read_ivar, gic_v3_read_ivar),
124 DEVMETHOD(bus_write_ivar, gic_v3_write_ivar),
126 /* Interrupt controller interface */
127 DEVMETHOD(pic_disable_intr, gic_v3_disable_intr),
128 DEVMETHOD(pic_enable_intr, gic_v3_enable_intr),
129 DEVMETHOD(pic_map_intr, gic_v3_map_intr),
130 DEVMETHOD(pic_setup_intr, gic_v3_setup_intr),
131 DEVMETHOD(pic_teardown_intr, gic_v3_teardown_intr),
132 DEVMETHOD(pic_post_filter, gic_v3_post_filter),
133 DEVMETHOD(pic_post_ithread, gic_v3_post_ithread),
134 DEVMETHOD(pic_pre_ithread, gic_v3_pre_ithread),
136 DEVMETHOD(pic_bind_intr, gic_v3_bind_intr),
137 DEVMETHOD(pic_init_secondary, gic_v3_init_secondary),
138 DEVMETHOD(pic_ipi_send, gic_v3_ipi_send),
139 DEVMETHOD(pic_ipi_setup, gic_v3_ipi_setup),
143 DEVMETHOD(msi_alloc_msi, gic_v3_alloc_msi),
144 DEVMETHOD(msi_release_msi, gic_v3_release_msi),
145 DEVMETHOD(msi_alloc_msix, gic_v3_alloc_msix),
146 DEVMETHOD(msi_release_msix, gic_v3_release_msix),
147 DEVMETHOD(msi_map_msi, gic_v3_map_msi),
150 DEVMETHOD(gic_reserve_msi_range, gic_v3_reserve_msi_range),
151 DEVMETHOD(gic_alloc_msi, gic_v3_gic_alloc_msi),
152 DEVMETHOD(gic_release_msi, gic_v3_gic_release_msi),
153 DEVMETHOD(gic_alloc_msix, gic_v3_gic_alloc_msix),
154 DEVMETHOD(gic_release_msix, gic_v3_gic_release_msix),
160 DEFINE_CLASS_0(gic, gic_v3_driver, gic_v3_methods,
161 sizeof(struct gic_v3_softc));
164 * Driver-specific definitions.
166 MALLOC_DEFINE(M_GIC_V3, "GICv3", GIC_V3_DEVSTR);
169 * Helper functions and definitions.
171 /* Destination registers, either Distributor or Re-Distributor */
177 struct gic_v3_irqsrc {
178 struct intr_irqsrc gi_isrc;
180 enum intr_polarity gi_pol;
181 enum intr_trigger gi_trig;
182 #define GI_FLAG_MSI (1 << 1) /* This interrupt source should only */
183 /* be used for MSI/MSI-X interrupts */
184 #define GI_FLAG_MSI_USED (1 << 2) /* This irq is already allocated */
185 /* for a MSI/MSI-X interrupt */
189 /* Helper routines starting with gic_v3_ */
190 static int gic_v3_dist_init(struct gic_v3_softc *);
191 static int gic_v3_redist_alloc(struct gic_v3_softc *);
192 static int gic_v3_redist_find(struct gic_v3_softc *);
193 static int gic_v3_redist_init(struct gic_v3_softc *);
194 static int gic_v3_cpu_init(struct gic_v3_softc *);
195 static void gic_v3_wait_for_rwp(struct gic_v3_softc *, enum gic_v3_xdist);
197 /* A sequence of init functions for primary (boot) CPU */
198 typedef int (*gic_v3_initseq_t) (struct gic_v3_softc *);
199 /* Primary CPU initialization sequence */
200 static gic_v3_initseq_t gic_v3_primary_init[] = {
209 /* Secondary CPU initialization sequence */
210 static gic_v3_initseq_t gic_v3_secondary_init[] = {
218 gic_r_read_4(device_t dev, bus_size_t offset)
220 struct gic_v3_softc *sc;
221 struct resource *rdist;
223 sc = device_get_softc(dev);
224 rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
225 return (bus_read_4(rdist, offset));
229 gic_r_read_8(device_t dev, bus_size_t offset)
231 struct gic_v3_softc *sc;
232 struct resource *rdist;
234 sc = device_get_softc(dev);
235 rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
236 return (bus_read_8(rdist, offset));
240 gic_r_write_4(device_t dev, bus_size_t offset, uint32_t val)
242 struct gic_v3_softc *sc;
243 struct resource *rdist;
245 sc = device_get_softc(dev);
246 rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
247 bus_write_4(rdist, offset, val);
251 gic_r_write_8(device_t dev, bus_size_t offset, uint64_t val)
253 struct gic_v3_softc *sc;
254 struct resource *rdist;
256 sc = device_get_softc(dev);
257 rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res;
258 bus_write_8(rdist, offset, val);
262 gic_v3_reserve_msi_range(device_t dev, u_int start, u_int count)
264 struct gic_v3_softc *sc;
267 sc = device_get_softc(dev);
269 KASSERT((start + count) < sc->gic_nirqs,
270 ("%s: Trying to allocate too many MSI IRQs: %d + %d > %d", __func__,
271 start, count, sc->gic_nirqs));
272 for (i = 0; i < count; i++) {
273 KASSERT(sc->gic_irqs[start + i].gi_isrc.isrc_handlers == 0,
274 ("%s: MSI interrupt %d already has a handler", __func__,
276 KASSERT(sc->gic_irqs[start + i].gi_pol == INTR_POLARITY_CONFORM,
277 ("%s: MSI interrupt %d already has a polarity", __func__,
279 KASSERT(sc->gic_irqs[start + i].gi_trig == INTR_TRIGGER_CONFORM,
280 ("%s: MSI interrupt %d already has a trigger", __func__,
282 sc->gic_irqs[start + i].gi_pol = INTR_POLARITY_HIGH;
283 sc->gic_irqs[start + i].gi_trig = INTR_TRIGGER_EDGE;
284 sc->gic_irqs[start + i].gi_flags |= GI_FLAG_MSI;
292 gic_v3_attach(device_t dev)
294 struct gic_v3_softc *sc;
295 gic_v3_initseq_t *init_func;
303 sc = device_get_softc(dev);
304 sc->gic_registered = FALSE;
308 /* Initialize mutex */
309 mtx_init(&sc->gic_mtx, "GICv3 lock", NULL, MTX_SPIN);
312 * Allocate array of struct resource.
313 * One entry for Distributor and all remaining for Re-Distributor.
315 sc->gic_res = malloc(
316 sizeof(*sc->gic_res) * (sc->gic_redists.nregions + 1),
319 /* Now allocate corresponding resources */
320 for (i = 0, rid = 0; i < (sc->gic_redists.nregions + 1); i++, rid++) {
321 sc->gic_res[rid] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
323 if (sc->gic_res[rid] == NULL)
328 * Distributor interface
330 sc->gic_dist = sc->gic_res[0];
333 * Re-Dristributor interface
335 /* Allocate space under region descriptions */
336 sc->gic_redists.regions = malloc(
337 sizeof(*sc->gic_redists.regions) * sc->gic_redists.nregions,
340 /* Fill-up bus_space information for each region. */
341 for (i = 0, rid = 1; i < sc->gic_redists.nregions; i++, rid++)
342 sc->gic_redists.regions[i] = sc->gic_res[rid];
344 /* Get the number of supported SPI interrupts */
345 typer = gic_d_read(sc, 4, GICD_TYPER);
346 sc->gic_nirqs = GICD_TYPER_I_NUM(typer);
347 if (sc->gic_nirqs > GIC_I_NUM_MAX)
348 sc->gic_nirqs = GIC_I_NUM_MAX;
350 sc->gic_irqs = malloc(sizeof(*sc->gic_irqs) * sc->gic_nirqs,
351 M_GIC_V3, M_WAITOK | M_ZERO);
352 name = device_get_nameunit(dev);
353 for (irq = 0; irq < sc->gic_nirqs; irq++) {
354 struct intr_irqsrc *isrc;
356 sc->gic_irqs[irq].gi_irq = irq;
357 sc->gic_irqs[irq].gi_pol = INTR_POLARITY_CONFORM;
358 sc->gic_irqs[irq].gi_trig = INTR_TRIGGER_CONFORM;
360 isrc = &sc->gic_irqs[irq].gi_isrc;
361 if (irq <= GIC_LAST_SGI) {
362 err = intr_isrc_register(isrc, sc->dev,
363 INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI);
364 } else if (irq <= GIC_LAST_PPI) {
365 err = intr_isrc_register(isrc, sc->dev,
366 INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI);
368 err = intr_isrc_register(isrc, sc->dev, 0,
369 "%s,s%u", name, irq - GIC_FIRST_SPI);
372 /* XXX call intr_isrc_deregister() */
373 free(sc->gic_irqs, M_DEVBUF);
378 mtx_init(&sc->gic_mbi_mtx, "GICv3 mbi lock", NULL, MTX_DEF);
379 if (sc->gic_mbi_start > 0) {
380 gic_v3_reserve_msi_range(dev, sc->gic_mbi_start,
381 sc->gic_mbi_end - sc->gic_mbi_start);
384 device_printf(dev, "using spi %u to %u\n", sc->gic_mbi_start,
390 * Read the Peripheral ID2 register. This is an implementation
391 * defined register, but seems to be implemented in all GICv3
392 * parts and Linux expects it to be there.
394 sc->gic_pidr2 = gic_d_read(sc, 4, GICD_PIDR2);
396 /* Get the number of supported interrupt identifier bits */
397 sc->gic_idbits = GICD_TYPER_IDBITS(typer);
400 device_printf(dev, "SPIs: %u, IDs: %u\n",
401 sc->gic_nirqs, (1 << sc->gic_idbits) - 1);
404 /* Train init sequence for boot CPU */
405 for (init_func = gic_v3_primary_init; *init_func != NULL; init_func++) {
406 err = (*init_func)(sc);
415 gic_v3_detach(device_t dev)
417 struct gic_v3_softc *sc;
421 sc = device_get_softc(dev);
423 if (device_is_attached(dev)) {
425 * XXX: We should probably deregister PIC
427 if (sc->gic_registered)
428 panic("Trying to detach registered PIC");
430 for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++)
431 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]);
433 for (i = 0; i <= mp_maxid; i++)
434 free(sc->gic_redists.pcpu[i], M_GIC_V3);
436 free(sc->gic_res, M_GIC_V3);
437 free(sc->gic_redists.regions, M_GIC_V3);
443 gic_v3_get_domain(device_t dev, device_t child, int *domain)
445 struct gic_v3_devinfo *di;
447 di = device_get_ivars(child);
448 if (di->gic_domain < 0)
451 *domain = di->gic_domain;
456 gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
458 struct gic_v3_softc *sc;
460 sc = device_get_softc(dev);
463 case GICV3_IVAR_NIRQS:
464 *result = (intr_nirq - sc->gic_nirqs) / sc->gic_nchildren;
466 case GICV3_IVAR_REDIST:
467 *result = (uintptr_t)sc->gic_redists.pcpu[PCPU_GET(cpuid)];
469 case GIC_IVAR_HW_REV:
471 GICR_PIDR2_ARCH(sc->gic_pidr2) == GICR_PIDR2_ARCH_GICv3 ||
472 GICR_PIDR2_ARCH(sc->gic_pidr2) == GICR_PIDR2_ARCH_GICv4,
473 ("gic_v3_read_ivar: Invalid GIC architecture: %d (%.08X)",
474 GICR_PIDR2_ARCH(sc->gic_pidr2), sc->gic_pidr2));
475 *result = GICR_PIDR2_ARCH(sc->gic_pidr2);
478 KASSERT(sc->gic_bus != GIC_BUS_UNKNOWN,
479 ("gic_v3_read_ivar: Unknown bus type"));
480 KASSERT(sc->gic_bus <= GIC_BUS_MAX,
481 ("gic_v3_read_ivar: Invalid bus type %u", sc->gic_bus));
482 *result = sc->gic_bus;
490 gic_v3_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
493 case GICV3_IVAR_NIRQS:
494 case GICV3_IVAR_REDIST:
495 case GIC_IVAR_HW_REV:
504 arm_gic_v3_intr(void *arg)
506 struct gic_v3_softc *sc = arg;
507 struct gic_v3_irqsrc *gi;
508 struct intr_pic *pic;
510 struct trapframe *tf;
515 if (CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1) {
517 * Hardware: Cavium ThunderX
518 * Chip revision: Pass 1.0 (early version)
519 * Pass 1.1 (production)
520 * ERRATUM: 22978, 23154
523 "nop;nop;nop;nop;nop;nop;nop;nop; \n"
524 "mrs %0, ICC_IAR1_EL1 \n"
525 "nop;nop;nop;nop; \n"
527 : "=&r" (active_irq));
529 active_irq = gic_icc_read(IAR1);
532 if (active_irq >= GIC_FIRST_LPI) {
533 intr_child_irq_handler(pic, active_irq);
537 if (__predict_false(active_irq >= sc->gic_nirqs))
538 return (FILTER_HANDLED);
540 tf = curthread->td_intr_frame;
541 gi = &sc->gic_irqs[active_irq];
542 if (active_irq <= GIC_LAST_SGI) {
543 /* Call EOI for all IPI before dispatch. */
544 gic_icc_write(EOIR1, (uint64_t)active_irq);
546 intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf);
548 device_printf(sc->dev, "SGI %ju on UP system detected\n",
549 (uintmax_t)(active_irq - GIC_FIRST_SGI));
551 } else if (active_irq >= GIC_FIRST_PPI &&
552 active_irq <= GIC_LAST_SPI) {
553 if (gi->gi_trig == INTR_TRIGGER_EDGE)
554 gic_icc_write(EOIR1, gi->gi_irq);
556 if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) {
557 if (gi->gi_trig != INTR_TRIGGER_EDGE)
558 gic_icc_write(EOIR1, gi->gi_irq);
559 gic_v3_disable_intr(sc->dev, &gi->gi_isrc);
560 device_printf(sc->dev,
561 "Stray irq %lu disabled\n", active_irq);
569 gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp,
570 enum intr_polarity *polp, enum intr_trigger *trigp)
578 * The 1st cell is the interrupt type:
581 * The 2nd cell contains the interrupt number:
584 * The 3rd cell is the flags, encoded as follows:
585 * bits[3:0] trigger type and level flags
587 * 2 = edge triggered (PPI only)
588 * 4 = level-sensitive
589 * 8 = level-sensitive (PPI only)
593 irq = GIC_FIRST_SPI + cells[1];
594 /* SPI irq is checked later. */
597 irq = GIC_FIRST_PPI + cells[1];
598 if (irq > GIC_LAST_PPI) {
599 device_printf(dev, "unsupported PPI interrupt "
600 "number %u\n", cells[1]);
605 device_printf(dev, "unsupported interrupt type "
606 "configuration %u\n", cells[0]);
610 switch (cells[2] & FDT_INTR_MASK) {
611 case FDT_INTR_EDGE_RISING:
612 *trigp = INTR_TRIGGER_EDGE;
613 *polp = INTR_POLARITY_HIGH;
615 case FDT_INTR_EDGE_FALLING:
616 *trigp = INTR_TRIGGER_EDGE;
617 *polp = INTR_POLARITY_LOW;
619 case FDT_INTR_LEVEL_HIGH:
620 *trigp = INTR_TRIGGER_LEVEL;
621 *polp = INTR_POLARITY_HIGH;
623 case FDT_INTR_LEVEL_LOW:
624 *trigp = INTR_TRIGGER_LEVEL;
625 *polp = INTR_POLARITY_LOW;
628 device_printf(dev, "unsupported trigger/polarity "
629 "configuration 0x%02x\n", cells[2]);
633 /* Check the interrupt is valid */
634 if (irq >= GIC_FIRST_SPI && *polp != INTR_POLARITY_HIGH)
643 gic_map_msi(device_t dev, struct intr_map_data_msi *msi_data, u_int *irqp,
644 enum intr_polarity *polp, enum intr_trigger *trigp)
646 struct gic_v3_irqsrc *gi;
649 gi = (struct gic_v3_irqsrc *)msi_data->isrc;
655 /* MSI/MSI-X interrupts are always edge triggered with high polarity */
656 *polp = INTR_POLARITY_HIGH;
657 *trigp = INTR_TRIGGER_EDGE;
663 do_gic_v3_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp,
664 enum intr_polarity *polp, enum intr_trigger *trigp)
666 struct gic_v3_softc *sc;
667 enum intr_polarity pol;
668 enum intr_trigger trig;
669 struct intr_map_data_msi *dam;
671 struct intr_map_data_fdt *daf;
674 struct intr_map_data_acpi *daa;
678 sc = device_get_softc(dev);
680 switch (data->type) {
682 case INTR_MAP_DATA_FDT:
683 daf = (struct intr_map_data_fdt *)data;
684 if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol,
690 case INTR_MAP_DATA_ACPI:
691 daa = (struct intr_map_data_acpi *)data;
697 case INTR_MAP_DATA_MSI:
699 dam = (struct intr_map_data_msi *)data;
700 if (gic_map_msi(dev, dam, &irq, &pol, &trig) != 0)
707 if (irq >= sc->gic_nirqs)
710 case INTR_POLARITY_CONFORM:
711 case INTR_POLARITY_LOW:
712 case INTR_POLARITY_HIGH:
718 case INTR_TRIGGER_CONFORM:
719 case INTR_TRIGGER_EDGE:
720 case INTR_TRIGGER_LEVEL:
735 gic_v3_map_intr(device_t dev, struct intr_map_data *data,
736 struct intr_irqsrc **isrcp)
738 struct gic_v3_softc *sc;
742 error = do_gic_v3_map_intr(dev, data, &irq, NULL, NULL);
744 sc = device_get_softc(dev);
745 *isrcp = GIC_INTR_ISRC(sc, irq);
751 gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc,
752 struct resource *res, struct intr_map_data *data)
754 struct gic_v3_softc *sc = device_get_softc(dev);
755 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
756 enum intr_trigger trig;
757 enum intr_polarity pol;
765 error = do_gic_v3_map_intr(dev, data, &irq, &pol, &trig);
769 if (gi->gi_irq != irq || pol == INTR_POLARITY_CONFORM ||
770 trig == INTR_TRIGGER_CONFORM)
773 /* Compare config if this is not first setup. */
774 if (isrc->isrc_handlers != 0) {
775 if (pol != gi->gi_pol || trig != gi->gi_trig)
781 /* For MSI/MSI-X we should have already configured these */
782 if ((gi->gi_flags & GI_FLAG_MSI) == 0) {
788 * XXX - In case that per CPU interrupt is going to be enabled in time
789 * when SMP is already started, we need some IPI call which
790 * enables it on others CPUs. Further, it's more complicated as
791 * pic_enable_source() and pic_disable_source() should act on
792 * per CPU basis only. Thus, it should be solved here somehow.
794 if (isrc->isrc_flags & INTR_ISRCF_PPI)
795 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
797 if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) {
798 mtx_lock_spin(&sc->gic_mtx);
800 /* Set the trigger and polarity */
801 if (irq <= GIC_LAST_PPI)
802 reg = gic_r_read(sc, 4,
803 GICR_SGI_BASE_SIZE + GICD_ICFGR(irq));
805 reg = gic_d_read(sc, 4, GICD_ICFGR(irq));
806 if (trig == INTR_TRIGGER_LEVEL)
807 reg &= ~(2 << ((irq % 16) * 2));
809 reg |= 2 << ((irq % 16) * 2);
811 if (irq <= GIC_LAST_PPI) {
813 GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg);
814 gic_v3_wait_for_rwp(sc, REDIST);
816 gic_d_write(sc, 4, GICD_ICFGR(irq), reg);
817 gic_v3_wait_for_rwp(sc, DIST);
820 mtx_unlock_spin(&sc->gic_mtx);
822 gic_v3_bind_intr(dev, isrc);
829 gic_v3_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
830 struct resource *res, struct intr_map_data *data)
832 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
834 if (isrc->isrc_handlers == 0 && (gi->gi_flags & GI_FLAG_MSI) == 0) {
835 gi->gi_pol = INTR_POLARITY_CONFORM;
836 gi->gi_trig = INTR_TRIGGER_CONFORM;
843 gic_v3_disable_intr(device_t dev, struct intr_irqsrc *isrc)
845 struct gic_v3_softc *sc;
846 struct gic_v3_irqsrc *gi;
849 sc = device_get_softc(dev);
850 gi = (struct gic_v3_irqsrc *)isrc;
853 if (irq <= GIC_LAST_PPI) {
854 /* SGIs and PPIs in corresponding Re-Distributor */
855 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq),
857 gic_v3_wait_for_rwp(sc, REDIST);
858 } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
859 /* SPIs in distributor */
860 gic_d_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq));
861 gic_v3_wait_for_rwp(sc, DIST);
863 panic("%s: Unsupported IRQ %u", __func__, irq);
867 gic_v3_enable_intr(device_t dev, struct intr_irqsrc *isrc)
869 struct gic_v3_softc *sc;
870 struct gic_v3_irqsrc *gi;
873 sc = device_get_softc(dev);
874 gi = (struct gic_v3_irqsrc *)isrc;
877 if (irq <= GIC_LAST_PPI) {
878 /* SGIs and PPIs in corresponding Re-Distributor */
879 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq),
881 gic_v3_wait_for_rwp(sc, REDIST);
882 } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
883 /* SPIs in distributor */
884 gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq));
885 gic_v3_wait_for_rwp(sc, DIST);
887 panic("%s: Unsupported IRQ %u", __func__, irq);
891 gic_v3_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
893 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
895 gic_v3_disable_intr(dev, isrc);
896 gic_icc_write(EOIR1, gi->gi_irq);
900 gic_v3_post_ithread(device_t dev, struct intr_irqsrc *isrc)
903 gic_v3_enable_intr(dev, isrc);
907 gic_v3_post_filter(device_t dev, struct intr_irqsrc *isrc)
909 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
911 if (gi->gi_trig == INTR_TRIGGER_EDGE)
914 gic_icc_write(EOIR1, gi->gi_irq);
918 gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc)
920 struct gic_v3_softc *sc;
921 struct gic_v3_irqsrc *gi;
924 gi = (struct gic_v3_irqsrc *)isrc;
925 if (gi->gi_irq <= GIC_LAST_PPI)
928 KASSERT(gi->gi_irq >= GIC_FIRST_SPI && gi->gi_irq <= GIC_LAST_SPI,
929 ("%s: Attempting to bind an invalid IRQ", __func__));
931 sc = device_get_softc(dev);
933 if (CPU_EMPTY(&isrc->isrc_cpu)) {
934 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus);
935 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu);
936 gic_d_write(sc, 8, GICD_IROUTER(gi->gi_irq),
937 CPU_AFFINITY(gic_irq_cpu));
940 * We can only bind to a single CPU so select
941 * the first CPU found.
943 cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
944 gic_d_write(sc, 8, GICD_IROUTER(gi->gi_irq), CPU_AFFINITY(cpu));
952 gic_v3_init_secondary(device_t dev)
955 struct gic_v3_softc *sc;
956 gic_v3_initseq_t *init_func;
957 struct intr_irqsrc *isrc;
961 sc = device_get_softc(dev);
962 cpu = PCPU_GET(cpuid);
964 /* Train init sequence for boot CPU */
965 for (init_func = gic_v3_secondary_init; *init_func != NULL;
967 err = (*init_func)(sc);
970 "Could not initialize GIC for CPU%u\n", cpu);
975 /* Unmask attached SGI interrupts. */
976 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) {
977 isrc = GIC_INTR_ISRC(sc, irq);
978 if (intr_isrc_init_on_cpu(isrc, cpu))
979 gic_v3_enable_intr(dev, isrc);
982 /* Unmask attached PPI interrupts. */
983 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) {
984 isrc = GIC_INTR_ISRC(sc, irq);
985 if (intr_isrc_init_on_cpu(isrc, cpu))
986 gic_v3_enable_intr(dev, isrc);
989 for (i = 0; i < sc->gic_nchildren; i++) {
990 child = sc->gic_children[i];
991 PIC_INIT_SECONDARY(child);
996 gic_v3_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
999 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
1000 uint64_t aff, val, irq;
1003 #define GIC_AFF_MASK (CPU_AFF3_MASK | CPU_AFF2_MASK | CPU_AFF1_MASK)
1004 #define GIC_AFFINITY(i) (CPU_AFFINITY(i) & GIC_AFF_MASK)
1005 aff = GIC_AFFINITY(0);
1009 /* Iterate through all CPUs in set */
1010 for (i = 0; i <= mp_maxid; i++) {
1011 /* Move to the next affinity group */
1012 if (aff != GIC_AFFINITY(i)) {
1015 gic_icc_write(SGI1R, val);
1018 aff = GIC_AFFINITY(i);
1021 /* Send the IPI to this cpu */
1022 if (CPU_ISSET(i, &cpus)) {
1023 #define ICC_SGI1R_AFFINITY(aff) \
1024 (((uint64_t)CPU_AFF3(aff) << ICC_SGI1R_EL1_AFF3_SHIFT) | \
1025 ((uint64_t)CPU_AFF2(aff) << ICC_SGI1R_EL1_AFF2_SHIFT) | \
1026 ((uint64_t)CPU_AFF1(aff) << ICC_SGI1R_EL1_AFF1_SHIFT))
1027 /* Set the affinity when the first at this level */
1029 val = ICC_SGI1R_AFFINITY(aff) |
1030 irq << ICC_SGI1R_EL1_SGIID_SHIFT;
1031 /* Set the bit to send the IPI to te CPU */
1032 val |= 1 << CPU_AFF0(CPU_AFFINITY(i));
1036 /* Send the IPI to the last cpu affinity group */
1038 gic_icc_write(SGI1R, val);
1044 gic_v3_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
1046 struct intr_irqsrc *isrc;
1047 struct gic_v3_softc *sc = device_get_softc(dev);
1049 if (sgi_first_unused > GIC_LAST_SGI)
1052 isrc = GIC_INTR_ISRC(sc, sgi_first_unused);
1053 sgi_to_ipi[sgi_first_unused++] = ipi;
1055 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
1066 gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist)
1068 struct resource *res;
1070 size_t us_left = 1000000;
1072 cpuid = PCPU_GET(cpuid);
1079 res = &sc->gic_redists.pcpu[cpuid]->res;
1082 KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__));
1086 while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) {
1089 panic("GICD Register write pending for too long");
1093 /* CPU interface. */
1094 static __inline void
1095 gic_v3_cpu_priority(uint64_t mask)
1098 /* Set prority mask */
1099 gic_icc_write(PMR, mask & ICC_PMR_EL1_PRIO_MASK);
1103 gic_v3_cpu_enable_sre(struct gic_v3_softc *sc)
1108 cpuid = PCPU_GET(cpuid);
1110 * Set the SRE bit to enable access to GIC CPU interface
1111 * via system registers.
1113 sre = READ_SPECIALREG(icc_sre_el1);
1114 sre |= ICC_SRE_EL1_SRE;
1115 WRITE_SPECIALREG(icc_sre_el1, sre);
1118 * Now ensure that the bit is set.
1120 sre = READ_SPECIALREG(icc_sre_el1);
1121 if ((sre & ICC_SRE_EL1_SRE) == 0) {
1122 /* We are done. This was disabled in EL2 */
1123 device_printf(sc->dev, "ERROR: CPU%u cannot enable CPU interface "
1124 "via system registers\n", cpuid);
1126 } else if (bootverbose) {
1127 device_printf(sc->dev,
1128 "CPU%u enabled CPU interface via system registers\n",
1136 gic_v3_cpu_init(struct gic_v3_softc *sc)
1140 /* Enable access to CPU interface via system registers */
1141 err = gic_v3_cpu_enable_sre(sc);
1144 /* Priority mask to minimum - accept all interrupts */
1145 gic_v3_cpu_priority(GIC_PRIORITY_MIN);
1146 /* Disable EOI mode */
1147 gic_icc_clear(CTLR, ICC_CTLR_EL1_EOIMODE);
1148 /* Enable group 1 (insecure) interrups */
1149 gic_icc_set(IGRPEN1, ICC_IGRPEN0_EL1_EN);
1156 gic_v3_dist_init(struct gic_v3_softc *sc)
1162 * 1. Disable the Distributor
1164 gic_d_write(sc, 4, GICD_CTLR, 0);
1165 gic_v3_wait_for_rwp(sc, DIST);
1168 * 2. Configure the Distributor
1170 /* Set all SPIs to be Group 1 Non-secure */
1171 for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_IGROUPRn)
1172 gic_d_write(sc, 4, GICD_IGROUPR(i), 0xFFFFFFFF);
1174 /* Set all global interrupts to be level triggered, active low. */
1175 for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ICFGRn)
1176 gic_d_write(sc, 4, GICD_ICFGR(i), 0x00000000);
1178 /* Set priority to all shared interrupts */
1179 for (i = GIC_FIRST_SPI;
1180 i < sc->gic_nirqs; i += GICD_I_PER_IPRIORITYn) {
1181 /* Set highest priority */
1182 gic_d_write(sc, 4, GICD_IPRIORITYR(i), GIC_PRIORITY_MAX);
1186 * Disable all interrupts. Leave PPI and SGIs as they are enabled in
1187 * Re-Distributor registers.
1189 for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ISENABLERn)
1190 gic_d_write(sc, 4, GICD_ICENABLER(i), 0xFFFFFFFF);
1192 gic_v3_wait_for_rwp(sc, DIST);
1195 * 3. Enable Distributor
1197 /* Enable Distributor with ARE, Group 1 */
1198 gic_d_write(sc, 4, GICD_CTLR, GICD_CTLR_ARE_NS | GICD_CTLR_G1A |
1202 * 4. Route all interrupts to boot CPU.
1204 aff = CPU_AFFINITY(0);
1205 for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i++)
1206 gic_d_write(sc, 8, GICD_IROUTER(i), aff);
1211 /* Re-Distributor */
1213 gic_v3_redist_alloc(struct gic_v3_softc *sc)
1217 /* Allocate struct resource for all CPU's Re-Distributor registers */
1218 for (cpuid = 0; cpuid <= mp_maxid; cpuid++)
1219 if (CPU_ISSET(cpuid, &all_cpus) != 0)
1220 sc->gic_redists.pcpu[cpuid] =
1221 malloc(sizeof(*sc->gic_redists.pcpu[0]),
1222 M_GIC_V3, M_WAITOK);
1224 sc->gic_redists.pcpu[cpuid] = NULL;
1229 gic_v3_redist_find(struct gic_v3_softc *sc)
1231 struct resource r_res;
1232 bus_space_handle_t r_bsh;
1239 cpuid = PCPU_GET(cpuid);
1241 aff = CPU_AFFINITY(cpuid);
1242 /* Affinity in format for comparison with typer */
1243 aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) |
1244 (CPU_AFF1(aff) << 8) | CPU_AFF0(aff);
1247 device_printf(sc->dev,
1248 "Start searching for Re-Distributor\n");
1250 /* Iterate through Re-Distributor regions */
1251 for (i = 0; i < sc->gic_redists.nregions; i++) {
1252 /* Take a copy of the region's resource */
1253 r_res = *sc->gic_redists.regions[i];
1254 r_bsh = rman_get_bushandle(&r_res);
1256 pidr2 = bus_read_4(&r_res, GICR_PIDR2);
1257 switch (GICR_PIDR2_ARCH(pidr2)) {
1258 case GICR_PIDR2_ARCH_GICv3: /* fall through */
1259 case GICR_PIDR2_ARCH_GICv4:
1262 device_printf(sc->dev,
1263 "No Re-Distributor found for CPU%u\n", cpuid);
1268 typer = bus_read_8(&r_res, GICR_TYPER);
1269 if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) {
1270 KASSERT(sc->gic_redists.pcpu[cpuid] != NULL,
1271 ("Invalid pointer to per-CPU redistributor"));
1272 /* Copy res contents to its final destination */
1273 sc->gic_redists.pcpu[cpuid]->res = r_res;
1274 sc->gic_redists.pcpu[cpuid]->lpi_enabled = false;
1276 device_printf(sc->dev,
1277 "CPU%u Re-Distributor has been found\n",
1283 r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE);
1284 if ((typer & GICR_TYPER_VLPIS) != 0) {
1286 (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE);
1289 rman_set_bushandle(&r_res, r_bsh);
1290 } while ((typer & GICR_TYPER_LAST) == 0);
1293 device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid);
1298 gic_v3_redist_wake(struct gic_v3_softc *sc)
1301 size_t us_left = 1000000;
1303 waker = gic_r_read(sc, 4, GICR_WAKER);
1304 /* Wake up Re-Distributor for this CPU */
1305 waker &= ~GICR_WAKER_PS;
1306 gic_r_write(sc, 4, GICR_WAKER, waker);
1308 * When clearing ProcessorSleep bit it is required to wait for
1309 * ChildrenAsleep to become zero following the processor power-on.
1311 while ((gic_r_read(sc, 4, GICR_WAKER) & GICR_WAKER_CA) != 0) {
1313 if (us_left-- == 0) {
1314 panic("Could not wake Re-Distributor for CPU%u",
1320 device_printf(sc->dev, "CPU%u Re-Distributor woke up\n",
1328 gic_v3_redist_init(struct gic_v3_softc *sc)
1333 err = gic_v3_redist_find(sc);
1337 err = gic_v3_redist_wake(sc);
1341 /* Configure SGIs and PPIs to be Group1 Non-secure */
1342 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_IGROUPR0,
1346 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ICENABLER0,
1347 GICR_I_ENABLER_PPI_MASK);
1349 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ISENABLER0,
1350 GICR_I_ENABLER_SGI_MASK);
1352 /* Set priority for SGIs and PPIs */
1353 for (i = 0; i <= GIC_LAST_PPI; i += GICR_I_PER_IPRIORITYn) {
1354 gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_IPRIORITYR(i),
1358 gic_v3_wait_for_rwp(sc, REDIST);
1364 * SPI-mapped Message Based Interrupts -- a GICv3 MSI/MSI-X controller.
1368 gic_v3_gic_alloc_msi(device_t dev, u_int mbi_start, u_int mbi_count,
1369 int count, int maxcount, struct intr_irqsrc **isrc)
1371 struct gic_v3_softc *sc;
1372 int i, irq, end_irq;
1375 KASSERT(powerof2(count), ("%s: bad count", __func__));
1376 KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__));
1378 sc = device_get_softc(dev);
1380 mtx_lock(&sc->gic_mbi_mtx);
1383 for (irq = mbi_start; irq < mbi_start + mbi_count; irq++) {
1384 /* Start on an aligned interrupt */
1385 if ((irq & (maxcount - 1)) != 0)
1388 /* Assume we found a valid range until shown otherwise */
1391 /* Check this range is valid */
1392 for (end_irq = irq; end_irq != irq + count; end_irq++) {
1393 /* No free interrupts */
1394 if (end_irq == mbi_start + mbi_count) {
1399 KASSERT((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI)!= 0,
1400 ("%s: Non-MSI interrupt found", __func__));
1402 /* This is already used */
1403 if ((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI_USED) ==
1413 /* Not enough interrupts were found */
1414 if (!found || irq == mbi_start + mbi_count) {
1415 mtx_unlock(&sc->gic_mbi_mtx);
1419 for (i = 0; i < count; i++) {
1420 /* Mark the interrupt as used */
1421 sc->gic_irqs[irq + i].gi_flags |= GI_FLAG_MSI_USED;
1423 mtx_unlock(&sc->gic_mbi_mtx);
1425 for (i = 0; i < count; i++)
1426 isrc[i] = (struct intr_irqsrc *)&sc->gic_irqs[irq + i];
1432 gic_v3_gic_release_msi(device_t dev, int count, struct intr_irqsrc **isrc)
1434 struct gic_v3_softc *sc;
1435 struct gic_v3_irqsrc *gi;
1438 sc = device_get_softc(dev);
1440 mtx_lock(&sc->gic_mbi_mtx);
1441 for (i = 0; i < count; i++) {
1442 gi = (struct gic_v3_irqsrc *)isrc[i];
1444 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED,
1445 ("%s: Trying to release an unused MSI-X interrupt",
1448 gi->gi_flags &= ~GI_FLAG_MSI_USED;
1450 mtx_unlock(&sc->gic_mbi_mtx);
1456 gic_v3_gic_alloc_msix(device_t dev, u_int mbi_start, u_int mbi_count,
1457 struct intr_irqsrc **isrcp)
1459 struct gic_v3_softc *sc;
1462 sc = device_get_softc(dev);
1464 mtx_lock(&sc->gic_mbi_mtx);
1465 /* Find an unused interrupt */
1466 for (irq = mbi_start; irq < mbi_start + mbi_count; irq++) {
1467 KASSERT((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) != 0,
1468 ("%s: Non-MSI interrupt found", __func__));
1469 if ((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == 0)
1472 /* No free interrupt was found */
1473 if (irq == mbi_start + mbi_count) {
1474 mtx_unlock(&sc->gic_mbi_mtx);
1478 /* Mark the interrupt as used */
1479 sc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI_USED;
1480 mtx_unlock(&sc->gic_mbi_mtx);
1482 *isrcp = (struct intr_irqsrc *)&sc->gic_irqs[irq];
1488 gic_v3_gic_release_msix(device_t dev, struct intr_irqsrc *isrc)
1490 struct gic_v3_softc *sc;
1491 struct gic_v3_irqsrc *gi;
1493 sc = device_get_softc(dev);
1494 gi = (struct gic_v3_irqsrc *)isrc;
1496 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED,
1497 ("%s: Trying to release an unused MSI-X interrupt", __func__));
1499 mtx_lock(&sc->gic_mbi_mtx);
1500 gi->gi_flags &= ~GI_FLAG_MSI_USED;
1501 mtx_unlock(&sc->gic_mbi_mtx);
1507 gic_v3_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1508 device_t *pic, struct intr_irqsrc **isrc)
1510 struct gic_v3_softc *sc;
1513 sc = device_get_softc(dev);
1514 error = gic_v3_gic_alloc_msi(dev, sc->gic_mbi_start,
1515 sc->gic_mbi_end - sc->gic_mbi_start, count, maxcount, isrc);
1524 gic_v3_release_msi(device_t dev, device_t child, int count,
1525 struct intr_irqsrc **isrc)
1527 return (gic_v3_gic_release_msi(dev, count, isrc));
1531 gic_v3_alloc_msix(device_t dev, device_t child, device_t *pic,
1532 struct intr_irqsrc **isrc)
1534 struct gic_v3_softc *sc;
1537 sc = device_get_softc(dev);
1538 error = gic_v3_gic_alloc_msix(dev, sc->gic_mbi_start,
1539 sc->gic_mbi_end - sc->gic_mbi_start, isrc);
1549 gic_v3_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1551 return (gic_v3_gic_release_msix(dev, isrc));
1555 gic_v3_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1556 uint64_t *addr, uint32_t *data)
1558 struct gic_v3_softc *sc = device_get_softc(dev);
1559 struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc;
1561 *addr = vtophys(rman_get_virtual(sc->gic_dist)) + GICD_SETSPI_NSR;