sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Andrew Turner under
   6  * the sponsorship of the FreeBSD Foundation.
   7  *
   8  * This software was developed by Semihalf under
   9  * the sponsorship of the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include "opt_acpi.h"
  34 #include "opt_platform.h"
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/bus.h>
  42 #include <sys/cpuset.h>
  43 #include <sys/endian.h>
  44 #include <sys/kernel.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/proc.h>
  48 #include <sys/queue.h>
  49 #include <sys/rman.h>
  50 #include <sys/smp.h>
  51 #include <sys/vmem.h>
  52
  53 #include <vm/vm.h>
  54 #include <vm/pmap.h>
  55
  56 #include <machine/bus.h>
  57 #include <machine/intr.h>
  58
  59 #include <arm/arm/gic_common.h>
  60 #include <arm64/arm64/gic_v3_reg.h>
  61 #include <arm64/arm64/gic_v3_var.h>
  62
  63 #ifdef FDT
  64 #include <dev/ofw/openfirm.h>
  65 #include <dev/ofw/ofw_bus.h>
  66 #include <dev/ofw/ofw_bus_subr.h>
  67 #endif
  68 #include <dev/pci/pcireg.h>
  69 #include <dev/pci/pcivar.h>
  70
  71 #include "pcib_if.h"
  72 #include "pic_if.h"
  73 #include "msi_if.h"
  74
  75 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  76     "ARM GICv3 Interrupt Translation Service");
  77
  78 #define LPI_NIRQS               (64 * 1024)
  79
  80 /* The size and alignment of the command circular buffer */
  81 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  82 #define ITS_CMDQ_ALIGN          (64 * 1024)
  83
  84 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  85 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  86 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  87
  88 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
  89 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
  90 #define LPI_PENDTAB_ALIGN       (64 * 1024)
  91 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  92
  93 #define LPI_INT_TRANS_TAB_ALIGN 256
  94 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
  95
  96 /* ITS commands encoding */
  97 #define ITS_CMD_MOVI            (0x01)
  98 #define ITS_CMD_SYNC            (0x05)
  99 #define ITS_CMD_MAPD            (0x08)
 100 #define ITS_CMD_MAPC            (0x09)
 101 #define ITS_CMD_MAPTI           (0x0a)
 102 #define ITS_CMD_MAPI            (0x0b)
 103 #define ITS_CMD_INV             (0x0c)
 104 #define ITS_CMD_INVALL          (0x0d)
 105 /* Command */
 106 #define CMD_COMMAND_MASK        (0xFFUL)
 107 /* PCI device ID */
 108 #define CMD_DEVID_SHIFT         (32)
 109 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 110 /* Size of IRQ ID bitfield */
 111 #define CMD_SIZE_MASK           (0xFFUL)
 112 /* Virtual LPI ID */
 113 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 114 /* Physical LPI ID */
 115 #define CMD_PID_SHIFT           (32)
 116 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 117 /* Collection */
 118 #define CMD_COL_MASK            (0xFFFFUL)
 119 /* Target (CPU or Re-Distributor) */
 120 #define CMD_TARGET_SHIFT        (16)
 121 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 122 /* Interrupt Translation Table address */
 123 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 124 /* Valid command bit */
 125 #define CMD_VALID_SHIFT         (63)
 126 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 127
 128 #define ITS_TARGET_NONE         0xFBADBEEF
 129
 130 /* LPI chunk owned by ITS device */
 131 struct lpi_chunk {
 132         u_int   lpi_base;
 133         u_int   lpi_free;       /* First free LPI in set */
 134         u_int   lpi_num;        /* Total number of LPIs in chunk */
 135         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 136 };
 137
 138 /* ITS device */
 139 struct its_dev {
 140         TAILQ_ENTRY(its_dev)    entry;
 141         /* PCI device */
 142         device_t                pci_dev;
 143         /* Device ID (i.e. PCI device ID) */
 144         uint32_t                devid;
 145         /* List of assigned LPIs */
 146         struct lpi_chunk        lpis;
 147         /* Virtual address of ITT */
 148         vm_offset_t             itt;
 149         size_t                  itt_size;
 150 };
 151
 152 /*
 153  * ITS command descriptor.
 154  * Idea for command description passing taken from Linux.
 155  */
 156 struct its_cmd_desc {
 157         uint8_t cmd_type;
 158
 159         union {
 160                 struct {
 161                         struct its_dev *its_dev;
 162                         struct its_col *col;
 163                         uint32_t id;
 164                 } cmd_desc_movi;
 165
 166                 struct {
 167                         struct its_col *col;
 168                 } cmd_desc_sync;
 169
 170                 struct {
 171                         struct its_col *col;
 172                         uint8_t valid;
 173                 } cmd_desc_mapc;
 174
 175                 struct {
 176                         struct its_dev *its_dev;
 177                         struct its_col *col;
 178                         uint32_t pid;
 179                         uint32_t id;
 180                 } cmd_desc_mapvi;
 181
 182                 struct {
 183                         struct its_dev *its_dev;
 184                         struct its_col *col;
 185                         uint32_t pid;
 186                 } cmd_desc_mapi;
 187
 188                 struct {
 189                         struct its_dev *its_dev;
 190                         uint8_t valid;
 191                 } cmd_desc_mapd;
 192
 193                 struct {
 194                         struct its_dev *its_dev;
 195                         struct its_col *col;
 196                         uint32_t pid;
 197                 } cmd_desc_inv;
 198
 199                 struct {
 200                         struct its_col *col;
 201                 } cmd_desc_invall;
 202         };
 203 };
 204
 205 /* ITS command. Each command is 32 bytes long */
 206 struct its_cmd {
 207         uint64_t        cmd_dword[4];   /* ITS command double word */
 208 };
 209
 210 /* An ITS private table */
 211 struct its_ptable {
 212         vm_offset_t     ptab_vaddr;
 213         unsigned long   ptab_size;
 214 };
 215
 216 /* ITS collection description. */
 217 struct its_col {
 218         uint64_t        col_target;     /* Target Re-Distributor */
 219         uint64_t        col_id;         /* Collection ID */
 220 };
 221
 222 struct gicv3_its_irqsrc {
 223         struct intr_irqsrc      gi_isrc;
 224         u_int                   gi_irq;
 225         struct its_dev          *gi_its_dev;
 226 };
 227
 228 struct gicv3_its_softc {
 229         struct intr_pic *sc_pic;
 230         struct resource *sc_its_res;
 231
 232         cpuset_t        sc_cpus;
 233         u_int           gic_irq_cpu;
 234
 235         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 236         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 237
 238         /*
 239          * TODO: We should get these from the parent as we only want a
 240          * single copy of each across the interrupt controller.
 241          */
 242         vm_offset_t sc_conf_base;
 243         vm_offset_t sc_pend_base[MAXCPU];
 244
 245         /* Command handling */
 246         struct mtx sc_its_cmd_lock;
 247         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 248         size_t sc_its_cmd_next_idx;
 249
 250         vmem_t *sc_irq_alloc;
 251         struct gicv3_its_irqsrc *sc_irqs;
 252         u_int   sc_irq_base;
 253         u_int   sc_irq_length;
 254
 255         struct mtx sc_its_dev_lock;
 256         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 257
 258 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 259 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 260 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 261         u_int sc_its_flags;
 262 };
 263
 264 typedef void (its_quirk_func_t)(device_t);
 265 static its_quirk_func_t its_quirk_cavium_22375;
 266
 267 static const struct {
 268         const char *desc;
 269         uint32_t iidr;
 270         uint32_t iidr_mask;
 271         its_quirk_func_t *func;
 272 } its_quirks[] = {
 273         {
 274                 /* Cavium ThunderX Pass 1.x */
 275                 .desc = "Cavoum ThunderX errata: 22375, 24313",
 276                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 277                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 278                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 279                 .func = its_quirk_cavium_22375,
 280         },
 281 };
 282
 283 #define gic_its_read_4(sc, reg)                 \
 284     bus_read_4((sc)->sc_its_res, (reg))
 285 #define gic_its_read_8(sc, reg)                 \
 286     bus_read_8((sc)->sc_its_res, (reg))
 287
 288 #define gic_its_write_4(sc, reg, val)           \
 289     bus_write_4((sc)->sc_its_res, (reg), (val))
 290 #define gic_its_write_8(sc, reg, val)           \
 291     bus_write_8((sc)->sc_its_res, (reg), (val))
 292
 293 static device_attach_t gicv3_its_attach;
 294 static device_detach_t gicv3_its_detach;
 295
 296 static pic_disable_intr_t gicv3_its_disable_intr;
 297 static pic_enable_intr_t gicv3_its_enable_intr;
 298 static pic_map_intr_t gicv3_its_map_intr;
 299 static pic_setup_intr_t gicv3_its_setup_intr;
 300 static pic_post_filter_t gicv3_its_post_filter;
 301 static pic_post_ithread_t gicv3_its_post_ithread;
 302 static pic_pre_ithread_t gicv3_its_pre_ithread;
 303 static pic_bind_intr_t gicv3_its_bind_intr;
 304 #ifdef SMP
 305 static pic_init_secondary_t gicv3_its_init_secondary;
 306 #endif
 307 static msi_alloc_msi_t gicv3_its_alloc_msi;
 308 static msi_release_msi_t gicv3_its_release_msi;
 309 static msi_alloc_msix_t gicv3_its_alloc_msix;
 310 static msi_release_msix_t gicv3_its_release_msix;
 311 static msi_map_msi_t gicv3_its_map_msi;
 312
 313 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 314 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 315 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 316 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 317 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 318 static void its_cmd_invall(device_t, struct its_col *);
 319
 320 static device_method_t gicv3_its_methods[] = {
 321         /* Device interface */
 322         DEVMETHOD(device_detach,        gicv3_its_detach),
 323
 324         /* Interrupt controller interface */
 325         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 326         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 327         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 328         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 329         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 330         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 331         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 332 #ifdef SMP
 333         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 334         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 335 #endif
 336
 337         /* MSI/MSI-X */
 338         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 339         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 340         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 341         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 342         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 343
 344         /* End */
 345         DEVMETHOD_END
 346 };
 347
 348 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 349     sizeof(struct gicv3_its_softc));
 350
 351 static void
 352 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 353 {
 354         vm_paddr_t cmd_paddr;
 355         uint64_t reg, tmp;
 356
 357         /* Set up the command circular buffer */
 358         sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS,
 359             M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0);
 360         sc->sc_its_cmd_next_idx = 0;
 361
 362         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 363
 364         /* Set the base of the command buffer */
 365         reg = GITS_CBASER_VALID |
 366             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 367             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 368             (ITS_CMDQ_SIZE / 4096 - 1);
 369         gic_its_write_8(sc, GITS_CBASER, reg);
 370
 371         /* Read back to check for fixed value fields */
 372         tmp = gic_its_read_8(sc, GITS_CBASER);
 373
 374         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 375             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 376                 /* Check if the hardware reported non-shareable */
 377                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 378                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 379                         /* If so remove the cache attribute */
 380                         reg &= ~GITS_CBASER_CACHE_MASK;
 381                         reg &= ~GITS_CBASER_SHARE_MASK;
 382                         /* Set to Non-cacheable, Non-shareable */
 383                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 384                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 385
 386                         gic_its_write_8(sc, GITS_CBASER, reg);
 387                 }
 388
 389                 /* The command queue has to be flushed after each command */
 390                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 391         }
 392
 393         /* Get the next command from the start of the buffer */
 394         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 395 }
 396
 397 static int
 398 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 399 {
 400         vm_offset_t table;
 401         vm_paddr_t paddr;
 402         uint64_t cache, reg, share, tmp, type;
 403         size_t esize, its_tbl_size, nidents, nitspages, npages;
 404         int i, page_size;
 405         int devbits;
 406
 407         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 408                 /*
 409                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 410                  * are to be 21 bits in length. The entry size of the ITS
 411                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 412                  * is supposed to be 8 bytes in length (for device table).
 413                  * Finally the page size that is to be used by ITS to access
 414                  * this table will be set to 64KB.
 415                  *
 416                  * This gives 0x200000 entries of size 0x8 bytes covered by
 417                  * 256 pages each of which 64KB in size. The number of pages
 418                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 419                  * that case this value would be 0xFF but on ThunderX the
 420                  * maximum value that HW accepts is 0xFD.
 421                  *
 422                  * Set an arbitrary number of device ID bits to 20 in order
 423                  * to limit the number of entries in ITS device table to
 424                  * 0x100000 and the table size to 8MB.
 425                  */
 426                 devbits = 20;
 427                 cache = 0;
 428         } else {
 429                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 430                 cache = GITS_BASER_CACHE_WAWB;
 431         }
 432         share = GITS_BASER_SHARE_IS;
 433         page_size = PAGE_SIZE_64K;
 434
 435         for (i = 0; i < GITS_BASER_NUM; i++) {
 436                 reg = gic_its_read_8(sc, GITS_BASER(i));
 437                 /* The type of table */
 438                 type = GITS_BASER_TYPE(reg);
 439                 /* The table entry size */
 440                 esize = GITS_BASER_ESIZE(reg);
 441
 442                 switch(type) {
 443                 case GITS_BASER_TYPE_DEV:
 444                         nidents = (1 << devbits);
 445                         its_tbl_size = esize * nidents;
 446                         its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K);
 447                         break;
 448                 case GITS_BASER_TYPE_VP:
 449                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 450                 case GITS_BASER_TYPE_IC:
 451                         its_tbl_size = page_size;
 452                         break;
 453                 default:
 454                         continue;
 455                 }
 456                 npages = howmany(its_tbl_size, PAGE_SIZE);
 457
 458                 /* Allocate the table */
 459                 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,
 460                     M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
 461                     PAGE_SIZE_64K, 0);
 462
 463                 sc->sc_its_ptab[i].ptab_vaddr = table;
 464                 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;
 465
 466                 paddr = vtophys(table);
 467
 468                 while (1) {
 469                         nitspages = howmany(its_tbl_size, page_size);
 470
 471                         /* Clear the fields we will be setting */
 472                         reg &= ~(GITS_BASER_VALID |
 473                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 474                             GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK |
 475                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 476                             GITS_BASER_SIZE_MASK);
 477                         /* Set the new values */
 478                         reg |= GITS_BASER_VALID |
 479                             (cache << GITS_BASER_CACHE_SHIFT) |
 480                             (type << GITS_BASER_TYPE_SHIFT) |
 481                             ((esize - 1) << GITS_BASER_ESIZE_SHIFT) |
 482                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 483                             (nitspages - 1);
 484
 485                         switch (page_size) {
 486                         case PAGE_SIZE:         /* 4KB */
 487                                 reg |=
 488                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 489                                 break;
 490                         case PAGE_SIZE_16K:     /* 16KB */
 491                                 reg |=
 492                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 493                                 break;
 494                         case PAGE_SIZE_64K:     /* 64KB */
 495                                 reg |=
 496                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 497                                 break;
 498                         }
 499
 500                         gic_its_write_8(sc, GITS_BASER(i), reg);
 501
 502                         /* Read back to check */
 503                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 504
 505                         /* Do the shareability masks line up? */
 506                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 507                             (reg & GITS_BASER_SHARE_MASK)) {
 508                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 509                                     GITS_BASER_SHARE_SHIFT;
 510                                 continue;
 511                         }
 512
 513                         if ((tmp & GITS_BASER_PSZ_MASK) !=
 514                             (reg & GITS_BASER_PSZ_MASK)) {
 515                                 switch (page_size) {
 516                                 case PAGE_SIZE_16K:
 517                                         page_size = PAGE_SIZE;
 518                                         continue;
 519                                 case PAGE_SIZE_64K:
 520                                         page_size = PAGE_SIZE_16K;
 521                                         continue;
 522                                 }
 523                         }
 524
 525                         if (tmp != reg) {
 526                                 device_printf(dev, "GITS_BASER%d: "
 527                                     "unable to be updated: %lx != %lx\n",
 528                                     i, reg, tmp);
 529                                 return (ENXIO);
 530                         }
 531
 532                         /* We should have made all needed changes */
 533                         break;
 534                 }
 535         }
 536
 537         return (0);
 538 }
 539
 540 static void
 541 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 542 {
 543
 544         sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE,
 545             M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN,
 546             0);
 547
 548         /* Set the default configuration */
 549         memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 550             LPI_CONFTAB_SIZE);
 551
 552         /* Flush the table to memory */
 553         cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE);
 554 }
 555
 556 static void
 557 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 558 {
 559         int i;
 560
 561         for (i = 0; i <= mp_maxid; i++) {
 562                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 563                         continue;
 564
 565                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 566                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 567                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 568
 569                 /* Flush so the ITS can see the memory */
 570                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
 571                     LPI_PENDTAB_SIZE);
 572         }
 573 }
 574
 575 static int
 576 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 577 {
 578         device_t gicv3;
 579         vm_paddr_t target;
 580         uint64_t xbaser, tmp;
 581         uint32_t ctlr;
 582         u_int cpuid;
 583         int domain;
 584
 585         if (!CPU_ISSET(PCPU_GET(cpuid), &sc->sc_cpus))
 586                 return (0);
 587
 588         if (bus_get_domain(dev, &domain) == 0) {
 589                 if (PCPU_GET(domain) != domain)
 590                         return (0);
 591         }
 592
 593         gicv3 = device_get_parent(dev);
 594         cpuid = PCPU_GET(cpuid);
 595
 596         /* Check if the ITS is enabled on this CPU */
 597         if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) {
 598                 return (ENXIO);
 599         }
 600
 601         /* Disable LPIs */
 602         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 603         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 604         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 605
 606         /* Make sure changes are observable my the GIC */
 607         dsb(sy);
 608
 609         /*
 610          * Set the redistributor base
 611          */
 612         xbaser = vtophys(sc->sc_conf_base) |
 613             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 614             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 615             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 616         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 617
 618         /* Check the cache attributes we set */
 619         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 620
 621         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 622             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 623                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 624                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 625                         /* We need to mark as non-cacheable */
 626                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 627                             GICR_PROPBASER_CACHE_MASK);
 628                         /* Non-cacheable */
 629                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 630                             GICR_PROPBASER_CACHE_SHIFT;
 631                         /* Non-sareable */
 632                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 633                             GICR_PROPBASER_SHARE_SHIFT;
 634                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 635                 }
 636                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 637         }
 638
 639         /*
 640          * Set the LPI pending table base
 641          */
 642         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 643             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 644             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 645
 646         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 647
 648         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 649
 650         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 651             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 652                 /* Clear the cahce and shareability bits */
 653                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 654                     GICR_PENDBASER_SHARE_MASK);
 655                 /* Mark as non-shareable */
 656                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 657                 /* And non-cacheable */
 658                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 659                     GICR_PENDBASER_CACHE_SHIFT;
 660         }
 661
 662         /* Enable LPIs */
 663         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 664         ctlr |= GICR_CTLR_LPI_ENABLE;
 665         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 666
 667         /* Make sure the GIC has seen everything */
 668         dsb(sy);
 669
 670         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 671                 /* This ITS wants the redistributor physical address */
 672                 target = vtophys(gicv3_get_redist_vaddr(dev));
 673         } else {
 674                 /* This ITS wants the unique processor number */
 675                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER));
 676         }
 677
 678         sc->sc_its_cols[cpuid]->col_target = target;
 679         sc->sc_its_cols[cpuid]->col_id = cpuid;
 680
 681         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 682         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 683
 684         return (0);
 685 }
 686
 687 static int
 688 gicv3_its_attach(device_t dev)
 689 {
 690         struct gicv3_its_softc *sc;
 691         const char *name;
 692         uint32_t iidr;
 693         int domain, err, i, rid;
 694
 695         sc = device_get_softc(dev);
 696
 697         sc->sc_irq_length = gicv3_get_nirqs(dev);
 698         sc->sc_irq_base = GIC_FIRST_LPI;
 699         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
 700
 701         rid = 0;
 702         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 703             RF_ACTIVE);
 704         if (sc->sc_its_res == NULL) {
 705                 device_printf(dev, "Could not allocate memory\n");
 706                 return (ENXIO);
 707         }
 708
 709         iidr = gic_its_read_4(sc, GITS_IIDR);
 710         for (i = 0; i < nitems(its_quirks); i++) {
 711                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 712                         if (bootverbose) {
 713                                 device_printf(dev, "Applying %s\n",
 714                                     its_quirks[i].desc);
 715                         }
 716                         its_quirks[i].func(dev);
 717                         break;
 718                 }
 719         }
 720
 721         /* Allocate the private tables */
 722         err = gicv3_its_table_init(dev, sc);
 723         if (err != 0)
 724                 return (err);
 725
 726         /* Protects access to the device list */
 727         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 728
 729         /* Protects access to the ITS command circular buffer. */
 730         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
 731
 732         if (bus_get_domain(dev, &domain) == 0) {
 733                 CPU_ZERO(&sc->sc_cpus);
 734                 if (domain < MAXMEMDOM)
 735                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
 736         } else {
 737                 CPU_COPY(&all_cpus, &sc->sc_cpus);
 738         }
 739
 740         /* Allocate the command circular buffer */
 741         gicv3_its_cmdq_init(sc);
 742
 743         /* Allocate the per-CPU collections */
 744         for (int cpu = 0; cpu <= mp_maxid; cpu++)
 745                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
 746                         sc->sc_its_cols[cpu] = malloc(
 747                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
 748                             M_WAITOK | M_ZERO);
 749                 else
 750                         sc->sc_its_cols[cpu] = NULL;
 751
 752         /* Enable the ITS */
 753         gic_its_write_4(sc, GITS_CTLR,
 754             gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN);
 755
 756         /* Create the LPI configuration table */
 757         gicv3_its_conftable_init(sc);
 758
 759         /* And the pending tebles */
 760         gicv3_its_pendtables_init(sc);
 761
 762         /* Enable LPIs on this CPU */
 763         its_init_cpu(dev, sc);
 764
 765         TAILQ_INIT(&sc->sc_its_dev_list);
 766
 767         /*
 768          * Create the vmem object to allocate INTRNG IRQs from. We try to
 769          * use all IRQs not already used by the GICv3.
 770          * XXX: This assumes there are no other interrupt controllers in the
 771          * system.
 772          */
 773         sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0,
 774             gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK);
 775
 776         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
 777             M_GICV3_ITS, M_WAITOK | M_ZERO);
 778         name = device_get_nameunit(dev);
 779         for (i = 0; i < sc->sc_irq_length; i++) {
 780                 sc->sc_irqs[i].gi_irq = i;
 781                 err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0,
 782                     "%s,%u", name, i);
 783         }
 784
 785         return (0);
 786 }
 787
 788 static int
 789 gicv3_its_detach(device_t dev)
 790 {
 791
 792         return (ENXIO);
 793 }
 794
 795 static void
 796 its_quirk_cavium_22375(device_t dev)
 797 {
 798         struct gicv3_its_softc *sc;
 799
 800         sc = device_get_softc(dev);
 801         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
 802 }
 803
 804 static void
 805 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
 806 {
 807         struct gicv3_its_softc *sc;
 808         struct gicv3_its_irqsrc *girq;
 809         uint8_t *conf;
 810
 811         sc = device_get_softc(dev);
 812         girq = (struct gicv3_its_irqsrc *)isrc;
 813         conf = (uint8_t *)sc->sc_conf_base;
 814
 815         conf[girq->gi_irq] &= ~LPI_CONF_ENABLE;
 816
 817         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
 818                 /* Clean D-cache under command. */
 819                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1);
 820         } else {
 821                 /* DSB inner shareable, store */
 822                 dsb(ishst);
 823         }
 824
 825         its_cmd_inv(dev, girq->gi_its_dev, girq);
 826 }
 827
 828 static void
 829 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
 830 {
 831         struct gicv3_its_softc *sc;
 832         struct gicv3_its_irqsrc *girq;
 833         uint8_t *conf;
 834
 835         sc = device_get_softc(dev);
 836         girq = (struct gicv3_its_irqsrc *)isrc;
 837         conf = (uint8_t *)sc->sc_conf_base;
 838
 839         conf[girq->gi_irq] |= LPI_CONF_ENABLE;
 840
 841         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
 842                 /* Clean D-cache under command. */
 843                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1);
 844         } else {
 845                 /* DSB inner shareable, store */
 846                 dsb(ishst);
 847         }
 848
 849         its_cmd_inv(dev, girq->gi_its_dev, girq);
 850 }
 851
 852 static int
 853 gicv3_its_intr(void *arg, uintptr_t irq)
 854 {
 855         struct gicv3_its_softc *sc = arg;
 856         struct gicv3_its_irqsrc *girq;
 857         struct trapframe *tf;
 858
 859         irq -= sc->sc_irq_base;
 860         girq = &sc->sc_irqs[irq];
 861         if (girq == NULL)
 862                 panic("gicv3_its_intr: Invalid interrupt %ld",
 863                     irq + sc->sc_irq_base);
 864
 865         tf = curthread->td_intr_frame;
 866         intr_isrc_dispatch(&girq->gi_isrc, tf);
 867         return (FILTER_HANDLED);
 868 }
 869
 870 static void
 871 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
 872 {
 873         struct gicv3_its_irqsrc *girq;
 874         struct gicv3_its_softc *sc;
 875
 876         sc = device_get_softc(dev);
 877         girq = (struct gicv3_its_irqsrc *)isrc;
 878         gicv3_its_disable_intr(dev, isrc);
 879         gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base);
 880 }
 881
 882 static void
 883 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
 884 {
 885
 886         gicv3_its_enable_intr(dev, isrc);
 887 }
 888
 889 static void
 890 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
 891 {
 892         struct gicv3_its_irqsrc *girq;
 893         struct gicv3_its_softc *sc;
 894
 895         sc = device_get_softc(dev);
 896         girq = (struct gicv3_its_irqsrc *)isrc;
 897         gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base);
 898 }
 899
 900 static int
 901 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
 902 {
 903         struct gicv3_its_irqsrc *girq;
 904         struct gicv3_its_softc *sc;
 905
 906         sc = device_get_softc(dev);
 907         girq = (struct gicv3_its_irqsrc *)isrc;
 908         if (CPU_EMPTY(&isrc->isrc_cpu)) {
 909                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
 910                     &sc->sc_cpus);
 911                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
 912         }
 913
 914         its_cmd_movi(dev, girq);
 915
 916         return (0);
 917 }
 918
 919 static int
 920 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
 921     struct intr_irqsrc **isrcp)
 922 {
 923
 924         /*
 925          * This should never happen, we only call this function to map
 926          * interrupts found before the controller driver is ready.
 927          */
 928         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
 929 }
 930
 931 static int
 932 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
 933     struct resource *res, struct intr_map_data *data)
 934 {
 935
 936         /* Bind the interrupt to a CPU */
 937         gicv3_its_bind_intr(dev, isrc);
 938
 939         return (0);
 940 }
 941
 942 #ifdef SMP
 943 static void
 944 gicv3_its_init_secondary(device_t dev)
 945 {
 946         struct gicv3_its_softc *sc;
 947
 948         sc = device_get_softc(dev);
 949
 950         /*
 951          * This is fatal as otherwise we may bind interrupts to this CPU.
 952          * We need a way to tell the interrupt framework to only bind to a
 953          * subset of given CPUs when it performs the shuffle.
 954          */
 955         if (its_init_cpu(dev, sc) != 0)
 956                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
 957                     PCPU_GET(cpuid));
 958 }
 959 #endif
 960
 961 static uint32_t
 962 its_get_devid(device_t pci_dev)
 963 {
 964         uintptr_t id;
 965
 966         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
 967                 panic("its_get_devid: Unable to get the MSI DeviceID");
 968
 969         return (id);
 970 }
 971
 972 static struct its_dev *
 973 its_device_find(device_t dev, device_t child)
 974 {
 975         struct gicv3_its_softc *sc;
 976         struct its_dev *its_dev = NULL;
 977
 978         sc = device_get_softc(dev);
 979
 980         mtx_lock_spin(&sc->sc_its_dev_lock);
 981         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
 982                 if (its_dev->pci_dev == child)
 983                         break;
 984         }
 985         mtx_unlock_spin(&sc->sc_its_dev_lock);
 986
 987         return (its_dev);
 988 }
 989
 990 static struct its_dev *
 991 its_device_get(device_t dev, device_t child, u_int nvecs)
 992 {
 993         struct gicv3_its_softc *sc;
 994         struct its_dev *its_dev;
 995         vmem_addr_t irq_base;
 996         size_t esize;
 997
 998         sc = device_get_softc(dev);
 999
1000         its_dev = its_device_find(dev, child);
1001         if (its_dev != NULL)
1002                 return (its_dev);
1003
1004         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1005         if (its_dev == NULL)
1006                 return (NULL);
1007
1008         its_dev->pci_dev = child;
1009         its_dev->devid = its_get_devid(child);
1010
1011         its_dev->lpis.lpi_busy = 0;
1012         its_dev->lpis.lpi_num = nvecs;
1013         its_dev->lpis.lpi_free = nvecs;
1014
1015         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1016             &irq_base) != 0) {
1017                 free(its_dev, M_GICV3_ITS);
1018                 return (NULL);
1019         }
1020         its_dev->lpis.lpi_base = irq_base;
1021
1022         /* Get ITT entry size */
1023         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1024
1025         /*
1026          * Allocate ITT for this device.
1027          * PA has to be 256 B aligned. At least two entries for device.
1028          */
1029         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1030         its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size,
1031             M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR,
1032             LPI_INT_TRANS_TAB_ALIGN, 0);
1033         if (its_dev->itt == 0) {
1034                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1035                 free(its_dev, M_GICV3_ITS);
1036                 return (NULL);
1037         }
1038
1039         mtx_lock_spin(&sc->sc_its_dev_lock);
1040         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1041         mtx_unlock_spin(&sc->sc_its_dev_lock);
1042
1043         /* Map device to its ITT */
1044         its_cmd_mapd(dev, its_dev, 1);
1045
1046         return (its_dev);
1047 }
1048
1049 static void
1050 its_device_release(device_t dev, struct its_dev *its_dev)
1051 {
1052         struct gicv3_its_softc *sc;
1053
1054         KASSERT(its_dev->lpis.lpi_busy == 0,
1055             ("its_device_release: Trying to release an inuse ITS device"));
1056
1057         /* Unmap device in ITS */
1058         its_cmd_mapd(dev, its_dev, 0);
1059
1060         sc = device_get_softc(dev);
1061
1062         /* Remove the device from the list of devices */
1063         mtx_lock_spin(&sc->sc_its_dev_lock);
1064         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1065         mtx_unlock_spin(&sc->sc_its_dev_lock);
1066
1067         /* Free ITT */
1068         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1069         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1070
1071         /* Free the IRQ allocation */
1072         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1073             its_dev->lpis.lpi_num);
1074
1075         free(its_dev, M_GICV3_ITS);
1076 }
1077
1078 static int
1079 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1080     device_t *pic, struct intr_irqsrc **srcs)
1081 {
1082         struct gicv3_its_softc *sc;
1083         struct gicv3_its_irqsrc *girq;
1084         struct its_dev *its_dev;
1085         u_int irq;
1086         int i;
1087
1088         its_dev = its_device_get(dev, child, count);
1089         if (its_dev == NULL)
1090                 return (ENXIO);
1091
1092         KASSERT(its_dev->lpis.lpi_free >= count,
1093             ("gicv3_its_alloc_msi: No free LPIs"));
1094         sc = device_get_softc(dev);
1095         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1096             its_dev->lpis.lpi_free;
1097         for (i = 0; i < count; i++, irq++) {
1098                 its_dev->lpis.lpi_free--;
1099                 girq = &sc->sc_irqs[irq];
1100                 girq->gi_its_dev = its_dev;
1101                 srcs[i] = (struct intr_irqsrc *)girq;
1102         }
1103         its_dev->lpis.lpi_busy += count;
1104         *pic = dev;
1105
1106         return (0);
1107 }
1108
1109 static int
1110 gicv3_its_release_msi(device_t dev, device_t child, int count,
1111     struct intr_irqsrc **isrc)
1112 {
1113         struct gicv3_its_irqsrc *girq;
1114         struct its_dev *its_dev;
1115         int i;
1116
1117         its_dev = its_device_find(dev, child);
1118
1119         KASSERT(its_dev != NULL,
1120             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1121              "no ITS device"));
1122         KASSERT(its_dev->lpis.lpi_busy >= count,
1123             ("gicv3_its_release_msi: Releasing more interrupts than "
1124              "were allocated: releasing %d, allocated %d", count,
1125              its_dev->lpis.lpi_busy));
1126         for (i = 0; i < count; i++) {
1127                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1128                 girq->gi_its_dev = NULL;
1129         }
1130         its_dev->lpis.lpi_busy -= count;
1131
1132         if (its_dev->lpis.lpi_busy == 0)
1133                 its_device_release(dev, its_dev);
1134
1135         return (0);
1136 }
1137
1138 static int
1139 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1140     struct intr_irqsrc **isrcp)
1141 {
1142         struct gicv3_its_softc *sc;
1143         struct gicv3_its_irqsrc *girq;
1144         struct its_dev *its_dev;
1145         u_int nvecs, irq;
1146
1147         nvecs = pci_msix_count(child);
1148         its_dev = its_device_get(dev, child, nvecs);
1149         if (its_dev == NULL)
1150                 return (ENXIO);
1151
1152         KASSERT(its_dev->lpis.lpi_free > 0,
1153             ("gicv3_its_alloc_msix: No free LPIs"));
1154         sc = device_get_softc(dev);
1155         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1156             its_dev->lpis.lpi_free;
1157         its_dev->lpis.lpi_free--;
1158         its_dev->lpis.lpi_busy++;
1159         girq = &sc->sc_irqs[irq];
1160         girq->gi_its_dev = its_dev;
1161
1162         *pic = dev;
1163         *isrcp = (struct intr_irqsrc *)girq;
1164
1165         return (0);
1166 }
1167
1168 static int
1169 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1170 {
1171         struct gicv3_its_irqsrc *girq;
1172         struct its_dev *its_dev;
1173
1174         its_dev = its_device_find(dev, child);
1175
1176         KASSERT(its_dev != NULL,
1177             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1178              "no ITS device"));
1179         KASSERT(its_dev->lpis.lpi_busy > 0,
1180             ("gicv3_its_release_msix: Releasing more interrupts than "
1181              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1182         girq = (struct gicv3_its_irqsrc *)isrc;
1183         girq->gi_its_dev = NULL;
1184         its_dev->lpis.lpi_busy--;
1185
1186         if (its_dev->lpis.lpi_busy == 0)
1187                 its_device_release(dev, its_dev);
1188
1189         return (0);
1190 }
1191
1192 static int
1193 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1194     uint64_t *addr, uint32_t *data)
1195 {
1196         struct gicv3_its_softc *sc;
1197         struct gicv3_its_irqsrc *girq;
1198
1199         sc = device_get_softc(dev);
1200         girq = (struct gicv3_its_irqsrc *)isrc;
1201
1202         /* Map the message to the given IRQ */
1203         its_cmd_mapti(dev, girq);
1204
1205         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1206         *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1207
1208         return (0);
1209 }
1210
1211 /*
1212  * Commands handling.
1213  */
1214
1215 static __inline void
1216 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1217 {
1218         /* Command field: DW0 [7:0] */
1219         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1220         cmd->cmd_dword[0] |= htole64(cmd_type);
1221 }
1222
1223 static __inline void
1224 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1225 {
1226         /* Device ID field: DW0 [63:32] */
1227         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1228         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1229 }
1230
1231 static __inline void
1232 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1233 {
1234         /* Size field: DW1 [4:0] */
1235         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1236         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1237 }
1238
1239 static __inline void
1240 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1241 {
1242         /* ID field: DW1 [31:0] */
1243         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1244         cmd->cmd_dword[1] |= htole64(id);
1245 }
1246
1247 static __inline void
1248 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1249 {
1250         /* Physical ID field: DW1 [63:32] */
1251         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1252         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1253 }
1254
1255 static __inline void
1256 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1257 {
1258         /* Collection field: DW2 [16:0] */
1259         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1260         cmd->cmd_dword[2] |= htole64(col_id);
1261 }
1262
1263 static __inline void
1264 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1265 {
1266         /* Target Address field: DW2 [47:16] */
1267         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1268         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1269 }
1270
1271 static __inline void
1272 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1273 {
1274         /* ITT Address field: DW2 [47:8] */
1275         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1276         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1277 }
1278
1279 static __inline void
1280 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1281 {
1282         /* Valid field: DW2 [63] */
1283         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1284         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1285 }
1286
1287 static inline bool
1288 its_cmd_queue_full(struct gicv3_its_softc *sc)
1289 {
1290         size_t read_idx, next_write_idx;
1291
1292         /* Get the index of the next command */
1293         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1294             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1295         /* And the index of the current command being read */
1296         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1297
1298         /*
1299          * The queue is full when the write offset points
1300          * at the command before the current read offset.
1301          */
1302         return (next_write_idx == read_idx);
1303 }
1304
1305 static inline void
1306 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1307 {
1308
1309         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1310                 /* Clean D-cache under command. */
1311                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1312         } else {
1313                 /* DSB inner shareable, store */
1314                 dsb(ishst);
1315         }
1316
1317 }
1318
1319 static inline uint64_t
1320 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1321 {
1322         uint64_t off;
1323
1324         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1325
1326         return (off);
1327 }
1328
1329 static void
1330 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1331     struct its_cmd *cmd_last)
1332 {
1333         struct gicv3_its_softc *sc;
1334         uint64_t first, last, read;
1335         size_t us_left;
1336
1337         sc = device_get_softc(dev);
1338
1339         /*
1340          * XXX ARM64TODO: This is obviously a significant delay.
1341          * The reason for that is that currently the time frames for
1342          * the command to complete are not known.
1343          */
1344         us_left = 1000000;
1345
1346         first = its_cmd_cwriter_offset(sc, cmd_first);
1347         last = its_cmd_cwriter_offset(sc, cmd_last);
1348
1349         for (;;) {
1350                 read = gic_its_read_8(sc, GITS_CREADR);
1351                 if (first < last) {
1352                         if (read < first || read >= last)
1353                                 break;
1354                 } else if (read < first && read >= last)
1355                         break;
1356
1357                 if (us_left-- == 0) {
1358                         /* This means timeout */
1359                         device_printf(dev,
1360                             "Timeout while waiting for CMD completion.\n");
1361                         return;
1362                 }
1363                 DELAY(1);
1364         }
1365 }
1366
1367
1368 static struct its_cmd *
1369 its_cmd_alloc_locked(device_t dev)
1370 {
1371         struct gicv3_its_softc *sc;
1372         struct its_cmd *cmd;
1373         size_t us_left;
1374
1375         sc = device_get_softc(dev);
1376
1377         /*
1378          * XXX ARM64TODO: This is obviously a significant delay.
1379          * The reason for that is that currently the time frames for
1380          * the command to complete (and therefore free the descriptor)
1381          * are not known.
1382          */
1383         us_left = 1000000;
1384
1385         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1386         while (its_cmd_queue_full(sc)) {
1387                 if (us_left-- == 0) {
1388                         /* Timeout while waiting for free command */
1389                         device_printf(dev,
1390                             "Timeout while waiting for free command\n");
1391                         return (NULL);
1392                 }
1393                 DELAY(1);
1394         }
1395
1396         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1397         sc->sc_its_cmd_next_idx++;
1398         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1399
1400         return (cmd);
1401 }
1402
1403 static uint64_t
1404 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1405 {
1406         uint64_t target;
1407         uint8_t cmd_type;
1408         u_int size;
1409
1410         cmd_type = desc->cmd_type;
1411         target = ITS_TARGET_NONE;
1412
1413         switch (cmd_type) {
1414         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1415                 target = desc->cmd_desc_movi.col->col_target;
1416                 cmd_format_command(cmd, ITS_CMD_MOVI);
1417                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1418                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1419                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1420                 break;
1421         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1422                 target = desc->cmd_desc_sync.col->col_target;
1423                 cmd_format_command(cmd, ITS_CMD_SYNC);
1424                 cmd_format_target(cmd, target);
1425                 break;
1426         case ITS_CMD_MAPD:      /* Assign ITT to device */
1427                 cmd_format_command(cmd, ITS_CMD_MAPD);
1428                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1429                 /*
1430                  * Size describes number of bits to encode interrupt IDs
1431                  * supported by the device minus one.
1432                  * When V (valid) bit is zero, this field should be written
1433                  * as zero.
1434                  */
1435                 if (desc->cmd_desc_mapd.valid != 0) {
1436                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1437                         size = MAX(1, size) - 1;
1438                 } else
1439                         size = 0;
1440
1441                 cmd_format_size(cmd, size);
1442                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1443                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1444                 break;
1445         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1446                 target = desc->cmd_desc_mapc.col->col_target;
1447                 cmd_format_command(cmd, ITS_CMD_MAPC);
1448                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1449                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1450                 cmd_format_target(cmd, target);
1451                 break;
1452         case ITS_CMD_MAPTI:
1453                 target = desc->cmd_desc_mapvi.col->col_target;
1454                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1455                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1456                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1457                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1458                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1459                 break;
1460         case ITS_CMD_MAPI:
1461                 target = desc->cmd_desc_mapi.col->col_target;
1462                 cmd_format_command(cmd, ITS_CMD_MAPI);
1463                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1464                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1465                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1466                 break;
1467         case ITS_CMD_INV:
1468                 target = desc->cmd_desc_inv.col->col_target;
1469                 cmd_format_command(cmd, ITS_CMD_INV);
1470                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1471                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1472                 break;
1473         case ITS_CMD_INVALL:
1474                 cmd_format_command(cmd, ITS_CMD_INVALL);
1475                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1476                 break;
1477         default:
1478                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1479         }
1480
1481         return (target);
1482 }
1483
1484 static int
1485 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1486 {
1487         struct gicv3_its_softc *sc;
1488         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1489         struct its_col col_sync;
1490         struct its_cmd_desc desc_sync;
1491         uint64_t target, cwriter;
1492
1493         sc = device_get_softc(dev);
1494         mtx_lock_spin(&sc->sc_its_cmd_lock);
1495         cmd = its_cmd_alloc_locked(dev);
1496         if (cmd == NULL) {
1497                 device_printf(dev, "could not allocate ITS command\n");
1498                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1499                 return (EBUSY);
1500         }
1501
1502         target = its_cmd_prepare(cmd, desc);
1503         its_cmd_sync(sc, cmd);
1504
1505         if (target != ITS_TARGET_NONE) {
1506                 cmd_sync = its_cmd_alloc_locked(dev);
1507                 if (cmd_sync != NULL) {
1508                         desc_sync.cmd_type = ITS_CMD_SYNC;
1509                         col_sync.col_target = target;
1510                         desc_sync.cmd_desc_sync.col = &col_sync;
1511                         its_cmd_prepare(cmd_sync, &desc_sync);
1512                         its_cmd_sync(sc, cmd_sync);
1513                 }
1514         }
1515
1516         /* Update GITS_CWRITER */
1517         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
1518         gic_its_write_8(sc, GITS_CWRITER, cwriter);
1519         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1520         mtx_unlock_spin(&sc->sc_its_cmd_lock);
1521
1522         its_cmd_wait_completion(dev, cmd, cmd_write);
1523
1524         return (0);
1525 }
1526
1527 /* Handlers to send commands */
1528 static void
1529 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
1530 {
1531         struct gicv3_its_softc *sc;
1532         struct its_cmd_desc desc;
1533         struct its_col *col;
1534
1535         sc = device_get_softc(dev);
1536         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1537
1538         desc.cmd_type = ITS_CMD_MOVI;
1539         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
1540         desc.cmd_desc_movi.col = col;
1541         desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1542
1543         its_cmd_send(dev, &desc);
1544 }
1545
1546 static void
1547 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
1548 {
1549         struct its_cmd_desc desc;
1550
1551         desc.cmd_type = ITS_CMD_MAPC;
1552         desc.cmd_desc_mapc.col = col;
1553         /*
1554          * Valid bit set - map the collection.
1555          * Valid bit cleared - unmap the collection.
1556          */
1557         desc.cmd_desc_mapc.valid = valid;
1558
1559         its_cmd_send(dev, &desc);
1560 }
1561
1562 static void
1563 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
1564 {
1565         struct gicv3_its_softc *sc;
1566         struct its_cmd_desc desc;
1567         struct its_col *col;
1568         u_int col_id;
1569
1570         sc = device_get_softc(dev);
1571
1572         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
1573         col = sc->sc_its_cols[col_id];
1574
1575         desc.cmd_type = ITS_CMD_MAPTI;
1576         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
1577         desc.cmd_desc_mapvi.col = col;
1578         /* The EventID sent to the device */
1579         desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1580         /* The physical interrupt presented to softeware */
1581         desc.cmd_desc_mapvi.pid = girq->gi_irq + sc->sc_irq_base;
1582
1583         its_cmd_send(dev, &desc);
1584 }
1585
1586 static void
1587 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
1588 {
1589         struct its_cmd_desc desc;
1590
1591         desc.cmd_type = ITS_CMD_MAPD;
1592         desc.cmd_desc_mapd.its_dev = its_dev;
1593         desc.cmd_desc_mapd.valid = valid;
1594
1595         its_cmd_send(dev, &desc);
1596 }
1597
1598 static void
1599 its_cmd_inv(device_t dev, struct its_dev *its_dev,
1600     struct gicv3_its_irqsrc *girq)
1601 {
1602         struct gicv3_its_softc *sc;
1603         struct its_cmd_desc desc;
1604         struct its_col *col;
1605
1606         sc = device_get_softc(dev);
1607         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1608
1609         desc.cmd_type = ITS_CMD_INV;
1610         /* The EventID sent to the device */
1611         desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base;
1612         desc.cmd_desc_inv.its_dev = its_dev;
1613         desc.cmd_desc_inv.col = col;
1614
1615         its_cmd_send(dev, &desc);
1616 }
1617
1618 static void
1619 its_cmd_invall(device_t dev, struct its_col *col)
1620 {
1621         struct its_cmd_desc desc;
1622
1623         desc.cmd_type = ITS_CMD_INVALL;
1624         desc.cmd_desc_invall.col = col;
1625
1626         its_cmd_send(dev, &desc);
1627 }
1628
1629 #ifdef FDT
1630 static device_probe_t gicv3_its_fdt_probe;
1631 static device_attach_t gicv3_its_fdt_attach;
1632
1633 static device_method_t gicv3_its_fdt_methods[] = {
1634         /* Device interface */
1635         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
1636         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
1637
1638         /* End */
1639         DEVMETHOD_END
1640 };
1641
1642 #define its_baseclasses its_fdt_baseclasses
1643 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
1644     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1645 #undef its_baseclasses
1646 static devclass_t gicv3_its_fdt_devclass;
1647
1648 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver,
1649     gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1650
1651 static int
1652 gicv3_its_fdt_probe(device_t dev)
1653 {
1654
1655         if (!ofw_bus_status_okay(dev))
1656                 return (ENXIO);
1657
1658         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
1659                 return (ENXIO);
1660
1661         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
1662         return (BUS_PROBE_DEFAULT);
1663 }
1664
1665 static int
1666 gicv3_its_fdt_attach(device_t dev)
1667 {
1668         struct gicv3_its_softc *sc;
1669         phandle_t xref;
1670         int err;
1671
1672         sc = device_get_softc(dev);
1673         err = gicv3_its_attach(dev);
1674         if (err != 0)
1675                 return (err);
1676
1677         /* Register this device as a interrupt controller */
1678         xref = OF_xref_from_node(ofw_bus_get_node(dev));
1679         sc->sc_pic = intr_pic_register(dev, xref);
1680         intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
1681             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
1682
1683         /* Register this device to handle MSI interrupts */
1684         intr_msi_register(dev, xref);
1685
1686         return (0);
1687 }
1688 #endif
1689
1690 #ifdef DEV_ACPI
1691 static device_probe_t gicv3_its_acpi_probe;
1692 static device_attach_t gicv3_its_acpi_attach;
1693
1694 static device_method_t gicv3_its_acpi_methods[] = {
1695         /* Device interface */
1696         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
1697         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
1698
1699         /* End */
1700         DEVMETHOD_END
1701 };
1702
1703 #define its_baseclasses its_acpi_baseclasses
1704 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
1705     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1706 #undef its_baseclasses
1707 static devclass_t gicv3_its_acpi_devclass;
1708
1709 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver,
1710     gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1711
1712 static int
1713 gicv3_its_acpi_probe(device_t dev)
1714 {
1715
1716         if (gic_get_bus(dev) != GIC_BUS_ACPI)
1717                 return (EINVAL);
1718
1719         if (gic_get_hw_rev(dev) < 3)
1720                 return (EINVAL);
1721
1722         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
1723         return (BUS_PROBE_DEFAULT);
1724 }
1725
1726 static int
1727 gicv3_its_acpi_attach(device_t dev)
1728 {
1729         struct gicv3_its_softc *sc;
1730         int err;
1731
1732         sc = device_get_softc(dev);
1733         err = gicv3_its_attach(dev);
1734         if (err != 0)
1735                 return (err);
1736
1737         sc->sc_pic = intr_pic_register(dev,
1738             device_get_unit(dev) + ACPI_MSI_XREF);
1739         intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
1740             gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS);
1741
1742         /* Register this device to handle MSI interrupts */
1743         intr_msi_register(dev, device_get_unit(dev) + ACPI_MSI_XREF);
1744
1745         return (0);
1746 }
1747 #endif