sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  * Copyright (c) 2023 Arm Ltd
   4  *
   5  * This software was developed by Andrew Turner under
   6  * the sponsorship of the FreeBSD Foundation.
   7  *
   8  * This software was developed by Semihalf under
   9  * the sponsorship of the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include "opt_acpi.h"
  34 #include "opt_platform.h"
  35 #include "opt_iommu.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/bus.h>
  40 #include <sys/cpuset.h>
  41 #include <sys/domainset.h>
  42 #include <sys/endian.h>
  43 #include <sys/kernel.h>
  44 #include <sys/lock.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/mutex.h>
  48 #include <sys/physmem.h>
  49 #include <sys/proc.h>
  50 #include <sys/taskqueue.h>
  51 #include <sys/tree.h>
  52 #include <sys/queue.h>
  53 #include <sys/rman.h>
  54 #include <sys/sbuf.h>
  55 #include <sys/smp.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/vmem.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #include <vm/vm_page.h>
  62
  63 #include <machine/bus.h>
  64 #include <machine/intr.h>
  65
  66 #include <arm/arm/gic_common.h>
  67 #include <arm64/arm64/gic_v3_reg.h>
  68 #include <arm64/arm64/gic_v3_var.h>
  69
  70 #ifdef FDT
  71 #include <dev/ofw/openfirm.h>
  72 #include <dev/ofw/ofw_bus.h>
  73 #include <dev/ofw/ofw_bus_subr.h>
  74 #endif
  75 #include <dev/pci/pcireg.h>
  76 #include <dev/pci/pcivar.h>
  77
  78 #ifdef IOMMU
  79 #include <dev/iommu/iommu.h>
  80 #include <dev/iommu/iommu_gas.h>
  81 #endif
  82
  83 #include "pcib_if.h"
  84 #include "pic_if.h"
  85 #include "msi_if.h"
  86
  87 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  88     "ARM GICv3 Interrupt Translation Service");
  89
  90 #define LPI_NIRQS               (64 * 1024)
  91
  92 /* The size and alignment of the command circular buffer */
  93 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  94 #define ITS_CMDQ_ALIGN          (64 * 1024)
  95
  96 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  97 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  98 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  99
 100 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
 101 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
 102 #define LPI_PENDTAB_ALIGN       (64 * 1024)
 103 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 104
 105 #define LPI_INT_TRANS_TAB_ALIGN 256
 106 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
 107
 108 /* ITS commands encoding */
 109 #define ITS_CMD_MOVI            (0x01)
 110 #define ITS_CMD_SYNC            (0x05)
 111 #define ITS_CMD_MAPD            (0x08)
 112 #define ITS_CMD_MAPC            (0x09)
 113 #define ITS_CMD_MAPTI           (0x0a)
 114 #define ITS_CMD_MAPI            (0x0b)
 115 #define ITS_CMD_INV             (0x0c)
 116 #define ITS_CMD_INVALL          (0x0d)
 117 /* Command */
 118 #define CMD_COMMAND_MASK        (0xFFUL)
 119 /* PCI device ID */
 120 #define CMD_DEVID_SHIFT         (32)
 121 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 122 /* Size of IRQ ID bitfield */
 123 #define CMD_SIZE_MASK           (0xFFUL)
 124 /* Virtual LPI ID */
 125 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 126 /* Physical LPI ID */
 127 #define CMD_PID_SHIFT           (32)
 128 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 129 /* Collection */
 130 #define CMD_COL_MASK            (0xFFFFUL)
 131 /* Target (CPU or Re-Distributor) */
 132 #define CMD_TARGET_SHIFT        (16)
 133 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 134 /* Interrupt Translation Table address */
 135 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 136 /* Valid command bit */
 137 #define CMD_VALID_SHIFT         (63)
 138 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 139
 140 #define ITS_TARGET_NONE         0xFBADBEEF
 141
 142 /* LPI chunk owned by ITS device */
 143 struct lpi_chunk {
 144         u_int   lpi_base;
 145         u_int   lpi_free;       /* First free LPI in set */
 146         u_int   lpi_num;        /* Total number of LPIs in chunk */
 147         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 148 };
 149
 150 /* ITS device */
 151 struct its_dev {
 152         TAILQ_ENTRY(its_dev)    entry;
 153         /* PCI device */
 154         device_t                pci_dev;
 155         /* Device ID (i.e. PCI device ID) */
 156         uint32_t                devid;
 157         /* List of assigned LPIs */
 158         struct lpi_chunk        lpis;
 159         /* Virtual address of ITT */
 160         vm_offset_t             itt;
 161         size_t                  itt_size;
 162 };
 163
 164 /*
 165  * ITS command descriptor.
 166  * Idea for command description passing taken from Linux.
 167  */
 168 struct its_cmd_desc {
 169         uint8_t cmd_type;
 170
 171         union {
 172                 struct {
 173                         struct its_dev *its_dev;
 174                         struct its_col *col;
 175                         uint32_t id;
 176                 } cmd_desc_movi;
 177
 178                 struct {
 179                         struct its_col *col;
 180                 } cmd_desc_sync;
 181
 182                 struct {
 183                         struct its_col *col;
 184                         uint8_t valid;
 185                 } cmd_desc_mapc;
 186
 187                 struct {
 188                         struct its_dev *its_dev;
 189                         struct its_col *col;
 190                         uint32_t pid;
 191                         uint32_t id;
 192                 } cmd_desc_mapvi;
 193
 194                 struct {
 195                         struct its_dev *its_dev;
 196                         struct its_col *col;
 197                         uint32_t pid;
 198                 } cmd_desc_mapi;
 199
 200                 struct {
 201                         struct its_dev *its_dev;
 202                         uint8_t valid;
 203                 } cmd_desc_mapd;
 204
 205                 struct {
 206                         struct its_dev *its_dev;
 207                         struct its_col *col;
 208                         uint32_t pid;
 209                 } cmd_desc_inv;
 210
 211                 struct {
 212                         struct its_col *col;
 213                 } cmd_desc_invall;
 214         };
 215 };
 216
 217 /* ITS command. Each command is 32 bytes long */
 218 struct its_cmd {
 219         uint64_t        cmd_dword[4];   /* ITS command double word */
 220 };
 221
 222 /* An ITS private table */
 223 struct its_ptable {
 224         vm_offset_t     ptab_vaddr;
 225         /* Size of the L1 and L2 tables */
 226         size_t          ptab_l1_size;
 227         size_t          ptab_l2_size;
 228         /* Number of L1 and L2 entries */
 229         int             ptab_l1_nidents;
 230         int             ptab_l2_nidents;
 231
 232         int             ptab_page_size;
 233         int             ptab_share;
 234         bool            ptab_indirect;
 235 };
 236
 237 /* ITS collection description. */
 238 struct its_col {
 239         uint64_t        col_target;     /* Target Re-Distributor */
 240         uint64_t        col_id;         /* Collection ID */
 241 };
 242
 243 struct gicv3_its_irqsrc {
 244         struct intr_irqsrc      gi_isrc;
 245         u_int                   gi_id;
 246         u_int                   gi_lpi;
 247         struct its_dev          *gi_its_dev;
 248         TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
 249 };
 250
 251 struct gicv3_its_softc {
 252         device_t        dev;
 253         struct intr_pic *sc_pic;
 254         struct resource *sc_its_res;
 255
 256         cpuset_t        sc_cpus;
 257         struct domainset *sc_ds;
 258         u_int           gic_irq_cpu;
 259         int             sc_devbits;
 260         int             sc_dev_table_idx;
 261
 262         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 263         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 264
 265         /*
 266          * TODO: We should get these from the parent as we only want a
 267          * single copy of each across the interrupt controller.
 268          */
 269         uint8_t         *sc_conf_base;
 270         vm_offset_t sc_pend_base[MAXCPU];
 271
 272         /* Command handling */
 273         struct mtx sc_its_cmd_lock;
 274         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 275         size_t sc_its_cmd_next_idx;
 276
 277         vmem_t *sc_irq_alloc;
 278         struct gicv3_its_irqsrc **sc_irqs;
 279         u_int   sc_irq_base;
 280         u_int   sc_irq_length;
 281         u_int   sc_irq_count;
 282
 283         struct mtx sc_its_dev_lock;
 284         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 285         TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
 286
 287 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 288 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 289 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 290 #define ITS_FLAGS_LPI_PREALLOC          0x00000008
 291         u_int sc_its_flags;
 292         bool    trace_enable;
 293         vm_page_t ma; /* fake msi page */
 294 };
 295
 296 typedef void (its_quirk_func_t)(device_t);
 297 static its_quirk_func_t its_quirk_cavium_22375;
 298
 299 static const struct {
 300         const char *desc;
 301         uint32_t iidr;
 302         uint32_t iidr_mask;
 303         its_quirk_func_t *func;
 304 } its_quirks[] = {
 305         {
 306                 /* Cavium ThunderX Pass 1.x */
 307                 .desc = "Cavium ThunderX errata: 22375, 24313",
 308                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 309                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 310                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 311                 .func = its_quirk_cavium_22375,
 312         },
 313 };
 314
 315 #define gic_its_read_4(sc, reg)                 \
 316     bus_read_4((sc)->sc_its_res, (reg))
 317 #define gic_its_read_8(sc, reg)                 \
 318     bus_read_8((sc)->sc_its_res, (reg))
 319
 320 #define gic_its_write_4(sc, reg, val)           \
 321     bus_write_4((sc)->sc_its_res, (reg), (val))
 322 #define gic_its_write_8(sc, reg, val)           \
 323     bus_write_8((sc)->sc_its_res, (reg), (val))
 324
 325 static device_attach_t gicv3_its_attach;
 326 static device_detach_t gicv3_its_detach;
 327
 328 static pic_disable_intr_t gicv3_its_disable_intr;
 329 static pic_enable_intr_t gicv3_its_enable_intr;
 330 static pic_map_intr_t gicv3_its_map_intr;
 331 static pic_setup_intr_t gicv3_its_setup_intr;
 332 static pic_post_filter_t gicv3_its_post_filter;
 333 static pic_post_ithread_t gicv3_its_post_ithread;
 334 static pic_pre_ithread_t gicv3_its_pre_ithread;
 335 static pic_bind_intr_t gicv3_its_bind_intr;
 336 #ifdef SMP
 337 static pic_init_secondary_t gicv3_its_init_secondary;
 338 #endif
 339 static msi_alloc_msi_t gicv3_its_alloc_msi;
 340 static msi_release_msi_t gicv3_its_release_msi;
 341 static msi_alloc_msix_t gicv3_its_alloc_msix;
 342 static msi_release_msix_t gicv3_its_release_msix;
 343 static msi_map_msi_t gicv3_its_map_msi;
 344 #ifdef IOMMU
 345 static msi_iommu_init_t gicv3_iommu_init;
 346 static msi_iommu_deinit_t gicv3_iommu_deinit;
 347 #endif
 348
 349 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 350 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 351 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 352 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 353 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 354 static void its_cmd_invall(device_t, struct its_col *);
 355
 356 static device_method_t gicv3_its_methods[] = {
 357         /* Device interface */
 358         DEVMETHOD(device_detach,        gicv3_its_detach),
 359
 360         /* Interrupt controller interface */
 361         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 362         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 363         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 364         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 365         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 366         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 367         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 368 #ifdef SMP
 369         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 370         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 371 #endif
 372
 373         /* MSI/MSI-X */
 374         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 375         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 376         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 377         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 378         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 379 #ifdef IOMMU
 380         DEVMETHOD(msi_iommu_init,       gicv3_iommu_init),
 381         DEVMETHOD(msi_iommu_deinit,     gicv3_iommu_deinit),
 382 #endif
 383
 384         /* End */
 385         DEVMETHOD_END
 386 };
 387
 388 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 389     sizeof(struct gicv3_its_softc));
 390
 391 static void
 392 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 393 {
 394         vm_paddr_t cmd_paddr;
 395         uint64_t reg, tmp;
 396
 397         /* Set up the command circular buffer */
 398         sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
 399             sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
 400             0);
 401         sc->sc_its_cmd_next_idx = 0;
 402
 403         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 404
 405         /* Set the base of the command buffer */
 406         reg = GITS_CBASER_VALID |
 407             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 408             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 409             (ITS_CMDQ_SIZE / 4096 - 1);
 410         gic_its_write_8(sc, GITS_CBASER, reg);
 411
 412         /* Read back to check for fixed value fields */
 413         tmp = gic_its_read_8(sc, GITS_CBASER);
 414
 415         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 416             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 417                 /* Check if the hardware reported non-shareable */
 418                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 419                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 420                         /* If so remove the cache attribute */
 421                         reg &= ~GITS_CBASER_CACHE_MASK;
 422                         reg &= ~GITS_CBASER_SHARE_MASK;
 423                         /* Set to Non-cacheable, Non-shareable */
 424                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 425                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 426
 427                         gic_its_write_8(sc, GITS_CBASER, reg);
 428                 }
 429
 430                 /* The command queue has to be flushed after each command */
 431                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 432         }
 433
 434         /* Get the next command from the start of the buffer */
 435         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 436 }
 437
 438 static int
 439 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
 440 {
 441         uint64_t reg, tmp;
 442         int page_size;
 443
 444         page_size = PAGE_SIZE_64K;
 445         reg = gic_its_read_8(sc, GITS_BASER(table));
 446
 447         while (1) {
 448                 reg &= GITS_BASER_PSZ_MASK;
 449                 switch (page_size) {
 450                 case PAGE_SIZE_4K:      /* 4KB */
 451                         reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 452                         break;
 453                 case PAGE_SIZE_16K:     /* 16KB */
 454                         reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 455                         break;
 456                 case PAGE_SIZE_64K:     /* 64KB */
 457                         reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 458                         break;
 459                 }
 460
 461                 /* Write the new page size */
 462                 gic_its_write_8(sc, GITS_BASER(table), reg);
 463
 464                 /* Read back to check */
 465                 tmp = gic_its_read_8(sc, GITS_BASER(table));
 466
 467                 /* The page size is correct */
 468                 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
 469                         return (page_size);
 470
 471                 switch (page_size) {
 472                 default:
 473                         return (-1);
 474                 case PAGE_SIZE_16K:
 475                         page_size = PAGE_SIZE_4K;
 476                         break;
 477                 case PAGE_SIZE_64K:
 478                         page_size = PAGE_SIZE_16K;
 479                         break;
 480                 }
 481         }
 482 }
 483
 484 static bool
 485 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table)
 486 {
 487         uint64_t reg;
 488
 489         reg = gic_its_read_8(sc, GITS_BASER(table));
 490
 491         /* Try setting the indirect flag */
 492         reg |= GITS_BASER_INDIRECT;
 493         gic_its_write_8(sc, GITS_BASER(table), reg);
 494
 495         /* Read back to check */
 496         reg = gic_its_read_8(sc, GITS_BASER(table));
 497         return ((reg & GITS_BASER_INDIRECT) != 0);
 498 }
 499
 500
 501 static int
 502 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 503 {
 504         vm_offset_t table;
 505         vm_paddr_t paddr;
 506         uint64_t cache, reg, share, tmp, type;
 507         size_t its_tbl_size, nitspages, npages;
 508         size_t l1_esize, l2_esize, l1_nidents, l2_nidents;
 509         int i, page_size;
 510         int devbits;
 511         bool indirect;
 512
 513         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 514                 /*
 515                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 516                  * are to be 21 bits in length. The entry size of the ITS
 517                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 518                  * is supposed to be 8 bytes in length (for device table).
 519                  * Finally the page size that is to be used by ITS to access
 520                  * this table will be set to 64KB.
 521                  *
 522                  * This gives 0x200000 entries of size 0x8 bytes covered by
 523                  * 256 pages each of which 64KB in size. The number of pages
 524                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 525                  * that case this value would be 0xFF but on ThunderX the
 526                  * maximum value that HW accepts is 0xFD.
 527                  *
 528                  * Set an arbitrary number of device ID bits to 20 in order
 529                  * to limit the number of entries in ITS device table to
 530                  * 0x100000 and the table size to 8MB.
 531                  */
 532                 devbits = 20;
 533                 cache = 0;
 534         } else {
 535                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 536                 cache = GITS_BASER_CACHE_WAWB;
 537         }
 538         sc->sc_devbits = devbits;
 539         share = GITS_BASER_SHARE_IS;
 540
 541         for (i = 0; i < GITS_BASER_NUM; i++) {
 542                 reg = gic_its_read_8(sc, GITS_BASER(i));
 543                 /* The type of table */
 544                 type = GITS_BASER_TYPE(reg);
 545                 if (type == GITS_BASER_TYPE_UNIMPL)
 546                         continue;
 547
 548                 /* The table entry size */
 549                 l1_esize = GITS_BASER_ESIZE(reg);
 550
 551                 /* Find the tables page size */
 552                 page_size = gicv3_its_table_page_size(sc, i);
 553                 if (page_size == -1) {
 554                         device_printf(dev, "No valid page size for table %d\n",
 555                             i);
 556                         return (EINVAL);
 557                 }
 558
 559                 indirect = false;
 560                 l2_nidents = 0;
 561                 l2_esize = 0;
 562                 switch(type) {
 563                 case GITS_BASER_TYPE_DEV:
 564                         if (sc->sc_dev_table_idx != -1)
 565                                 device_printf(dev,
 566                                     "Warning: Multiple device tables found\n");
 567
 568                         sc->sc_dev_table_idx = i;
 569                         l1_nidents = (1 << devbits);
 570                         if ((l1_esize * l1_nidents) > (page_size * 2)) {
 571                                 indirect =
 572                                     gicv3_its_table_supports_indirect(sc, i);
 573                                 if (indirect) {
 574                                         /*
 575                                          * Each l1 entry is 8 bytes and points
 576                                          * to an l2 table of size page_size.
 577                                          * Calculate how many entries this is
 578                                          * and use this to find how many
 579                                          * 8 byte l1 idents we need.
 580                                          */
 581                                         l2_esize = l1_esize;
 582                                         l2_nidents = page_size / l2_esize;
 583                                         l1_nidents = l1_nidents / l2_nidents;
 584                                         l1_esize = GITS_INDIRECT_L1_ESIZE;
 585                                 }
 586                         }
 587                         its_tbl_size = l1_esize * l1_nidents;
 588                         its_tbl_size = roundup2(its_tbl_size, page_size);
 589                         break;
 590                 case GITS_BASER_TYPE_VP:
 591                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 592                 case GITS_BASER_TYPE_IC:
 593                         its_tbl_size = page_size;
 594                         break;
 595                 default:
 596                         if (bootverbose)
 597                                 device_printf(dev, "Unhandled table type %lx\n",
 598                                     type);
 599                         continue;
 600                 }
 601                 npages = howmany(its_tbl_size, PAGE_SIZE);
 602
 603                 /* Allocate the table */
 604                 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
 605                     M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
 606                     (1ul << 48) - 1, PAGE_SIZE_64K, 0);
 607
 608                 sc->sc_its_ptab[i].ptab_vaddr = table;
 609                 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size;
 610                 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents;
 611                 sc->sc_its_ptab[i].ptab_l2_size = page_size;
 612                 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents;
 613
 614                 sc->sc_its_ptab[i].ptab_indirect = indirect;
 615                 sc->sc_its_ptab[i].ptab_page_size = page_size;
 616
 617                 paddr = vtophys(table);
 618
 619                 while (1) {
 620                         nitspages = howmany(its_tbl_size, page_size);
 621
 622                         /* Clear the fields we will be setting */
 623                         reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
 624                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 625                             GITS_BASER_PA_MASK |
 626                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 627                             GITS_BASER_SIZE_MASK);
 628                         /* Set the new values */
 629                         reg |= GITS_BASER_VALID |
 630                             (indirect ? GITS_BASER_INDIRECT : 0) |
 631                             (cache << GITS_BASER_CACHE_SHIFT) |
 632                             (type << GITS_BASER_TYPE_SHIFT) |
 633                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 634                             (nitspages - 1);
 635
 636                         switch (page_size) {
 637                         case PAGE_SIZE_4K:      /* 4KB */
 638                                 reg |=
 639                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 640                                 break;
 641                         case PAGE_SIZE_16K:     /* 16KB */
 642                                 reg |=
 643                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 644                                 break;
 645                         case PAGE_SIZE_64K:     /* 64KB */
 646                                 reg |=
 647                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 648                                 break;
 649                         }
 650
 651                         gic_its_write_8(sc, GITS_BASER(i), reg);
 652
 653                         /* Read back to check */
 654                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 655
 656                         /* Do the shareability masks line up? */
 657                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 658                             (reg & GITS_BASER_SHARE_MASK)) {
 659                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 660                                     GITS_BASER_SHARE_SHIFT;
 661                                 continue;
 662                         }
 663
 664                         if (tmp != reg) {
 665                                 device_printf(dev, "GITS_BASER%d: "
 666                                     "unable to be updated: %lx != %lx\n",
 667                                     i, reg, tmp);
 668                                 return (ENXIO);
 669                         }
 670
 671                         sc->sc_its_ptab[i].ptab_share = share;
 672                         /* We should have made all needed changes */
 673                         break;
 674                 }
 675         }
 676
 677         return (0);
 678 }
 679
 680 static void
 681 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 682 {
 683         /* note: we assume the ITS children are serialized by the parent */
 684         static void *conf_table;
 685         int extra_flags = 0;
 686         device_t gicv3;
 687         uint32_t ctlr;
 688         vm_paddr_t conf_pa;
 689         vm_offset_t conf_va;
 690
 691         /*
 692          * The PROPBASER is a singleton in our parent. We only set it up the
 693          * first time through. conf_table is effectively global to all the units
 694          * and we rely on subr_bus to serialize probe/attach.
 695          */
 696         if (conf_table != NULL) {
 697                 sc->sc_conf_base = conf_table;
 698                 return;
 699         }
 700
 701         gicv3 = device_get_parent(sc->dev);
 702         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 703         if ((ctlr & GICR_CTLR_LPI_ENABLE) != 0) {
 704                 conf_pa = gic_r_read_8(gicv3, GICR_PROPBASER);
 705                 conf_pa &= GICR_PROPBASER_PA_MASK;
 706                 /*
 707                  * If there was a pre-existing PROPBASER, then we need to honor
 708                  * it because implemenetation defined behavior in gicv3 makes it
 709                  * impossible to quiesce to change it out. We will only see a
 710                  * pre-existing one when we've been kexec'd from a Linux kernel,
 711                  * or from a LinuxBoot environment.
 712                  *
 713                  * Linux provides us with a MEMRESERVE table that we put into
 714                  * the excluded physmem area. If PROPBASER isn't in this tabke,
 715                  * the system cannot run due to random memory corruption,
 716                  * so we panic for this case.
 717                  */
 718                 if (!physmem_excluded(conf_pa, LPI_CONFTAB_SIZE))
 719                         panic("gicv3 PROPBASER needs to reuse %#lx, but not reserved\n",
 720                             conf_pa);
 721                 conf_va = PHYS_TO_DMAP(conf_pa);
 722                 if (!pmap_klookup(conf_va, NULL))
 723                         panic("Can't mapped prior LPI mapping into VA\n");
 724                 conf_table = (void *)conf_va;
 725                 extra_flags = ITS_FLAGS_LPI_PREALLOC | ITS_FLAGS_LPI_CONF_FLUSH;
 726                 if (bootverbose)
 727                         device_printf(sc->dev,
 728                             "LPI enabled, conf table using pa %#lx va %lx\n",
 729                             conf_pa, conf_va);
 730         } else {
 731
 732                 /*
 733                  * Otherwise just allocate contiguous pages. We'll configure the
 734                  * PROPBASER register later in its_init_cpu_lpi().
 735                  */
 736                 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
 737                     M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
 738                     LPI_CONFTAB_ALIGN, 0);
 739         }
 740         sc->sc_conf_base = conf_table;
 741         sc->sc_its_flags |= extra_flags;
 742
 743         /* Set the default configuration */
 744         memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 745             LPI_CONFTAB_SIZE);
 746
 747         /* Flush the table to memory */
 748         cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
 749 }
 750
 751 static void
 752 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 753 {
 754
 755         if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
 756                 for (int i = 0; i <= mp_maxid; i++) {
 757                         if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 758                                 continue;
 759
 760                         sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 761                             LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 762                             0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 763
 764                         /* Flush so the ITS can see the memory */
 765                         cpu_dcache_wb_range(sc->sc_pend_base[i],
 766                             LPI_PENDTAB_SIZE);
 767                 }
 768         }
 769 }
 770
 771 static void
 772 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
 773 {
 774         device_t gicv3;
 775         uint64_t xbaser, tmp, size;
 776         uint32_t ctlr;
 777         u_int cpuid;
 778
 779         gicv3 = device_get_parent(dev);
 780         cpuid = PCPU_GET(cpuid);
 781
 782         /*
 783          * Set the redistributor base. If we're reusing what we found on boot
 784          * since the gic was already running, then don't touch it here. We also
 785          * don't need to disable / enable LPI if we're not changing PROPBASER,
 786          * so only do that if we're not prealloced.
 787          */
 788         if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
 789                 /* Disable LPIs */
 790                 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 791                 ctlr &= ~GICR_CTLR_LPI_ENABLE;
 792                 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 793
 794                 /* Make sure changes are observable my the GIC */
 795                 dsb(sy);
 796
 797                 size = (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 798
 799                 xbaser = vtophys(sc->sc_conf_base) |
 800                     (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 801                     (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 802                     size;
 803
 804                 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 805
 806                 /* Check the cache attributes we set */
 807                 tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 808
 809                 if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 810                     (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 811                         if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 812                             (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 813                                 /* We need to mark as non-cacheable */
 814                                 xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 815                                     GICR_PROPBASER_CACHE_MASK);
 816                                 /* Non-cacheable */
 817                                 xbaser |= GICR_PROPBASER_CACHE_NIN <<
 818                                     GICR_PROPBASER_CACHE_SHIFT;
 819                                 /* Non-shareable */
 820                                 xbaser |= GICR_PROPBASER_SHARE_NS <<
 821                                     GICR_PROPBASER_SHARE_SHIFT;
 822                                 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 823                         }
 824                         sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 825                 }
 826
 827                 /*
 828                  * Set the LPI pending table base
 829                  */
 830                 xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 831                     (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 832                     (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 833
 834                 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 835
 836                 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 837
 838                 if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 839                     (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 840                         /* Clear the cahce and shareability bits */
 841                         xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 842                             GICR_PENDBASER_SHARE_MASK);
 843                         /* Mark as non-shareable */
 844                         xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 845                         /* And non-cacheable */
 846                         xbaser |= GICR_PENDBASER_CACHE_NIN <<
 847                             GICR_PENDBASER_CACHE_SHIFT;
 848                 }
 849
 850                 /* Enable LPIs */
 851                 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 852                 ctlr |= GICR_CTLR_LPI_ENABLE;
 853                 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 854
 855                 /* Make sure the GIC has seen everything */
 856                 dsb(sy);
 857         } else {
 858                 KASSERT(sc->sc_pend_base[cpuid] == 0,
 859                     ("PREALLOC too soon cpuid %d", cpuid));
 860                 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 861                 tmp &= GICR_PENDBASER_PA_MASK;
 862                 if (!physmem_excluded(tmp, LPI_PENDTAB_SIZE))
 863                         panic("gicv3 PENDBASER on cpu %d needs to reuse 0x%#lx, but not reserved\n",
 864                             cpuid, tmp);
 865                 sc->sc_pend_base[cpuid] = PHYS_TO_DMAP(tmp);
 866         }
 867
 868
 869         if (bootverbose)
 870                 device_printf(gicv3, "using %sPENDBASE of %#lx on cpu %d\n",
 871                     (sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) ? "pre-existing " : "",
 872                     vtophys(sc->sc_pend_base[cpuid]), cpuid);
 873 }
 874
 875 static int
 876 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 877 {
 878         device_t gicv3;
 879         vm_paddr_t target;
 880         u_int cpuid;
 881         struct redist_pcpu *rpcpu;
 882
 883         gicv3 = device_get_parent(dev);
 884         cpuid = PCPU_GET(cpuid);
 885         if (!CPU_ISSET(cpuid, &sc->sc_cpus))
 886                 return (0);
 887
 888         /* Check if the ITS is enabled on this CPU */
 889         if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
 890                 return (ENXIO);
 891
 892         rpcpu = gicv3_get_redist(dev);
 893
 894         /* Do per-cpu LPI init once */
 895         if (!rpcpu->lpi_enabled) {
 896                 its_init_cpu_lpi(dev, sc);
 897                 rpcpu->lpi_enabled = true;
 898         }
 899
 900         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 901                 /* This ITS wants the redistributor physical address */
 902                 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
 903                     rpcpu->offset);
 904         } else {
 905                 /* This ITS wants the unique processor number */
 906                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
 907                     CMD_TARGET_SHIFT;
 908         }
 909
 910         sc->sc_its_cols[cpuid]->col_target = target;
 911         sc->sc_its_cols[cpuid]->col_id = cpuid;
 912
 913         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 914         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 915
 916         return (0);
 917 }
 918
 919 static int
 920 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
 921 {
 922         struct gicv3_its_softc *sc;
 923         int rv;
 924
 925         sc = arg1;
 926
 927         rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
 928         if (rv != 0 || req->newptr == NULL)
 929                 return (rv);
 930         if (sc->trace_enable)
 931                 gic_its_write_8(sc, GITS_TRKCTLR, 3);
 932         else
 933                 gic_its_write_8(sc, GITS_TRKCTLR, 0);
 934
 935         return (0);
 936 }
 937
 938 static int
 939 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
 940 {
 941         struct gicv3_its_softc *sc;
 942         struct sbuf *sb;
 943         int err;
 944
 945         sc = arg1;
 946         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 947         if (sb == NULL) {
 948                 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
 949                 return (ENOMEM);
 950         }
 951         sbuf_cat(sb, "\n");
 952         sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
 953             gic_its_read_4(sc, GITS_TRKCTLR));
 954         sbuf_printf(sb, "GITS_TRKR:    0x%08X\n",
 955             gic_its_read_4(sc, GITS_TRKR));
 956         sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
 957             gic_its_read_4(sc, GITS_TRKDIDR));
 958         sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
 959             gic_its_read_4(sc, GITS_TRKPIDR));
 960         sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
 961             gic_its_read_4(sc, GITS_TRKVIDR));
 962         sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
 963            gic_its_read_4(sc, GITS_TRKTGTR));
 964
 965         err = sbuf_finish(sb);
 966         if (err)
 967                 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
 968         sbuf_delete(sb);
 969         return(err);
 970 }
 971
 972 static int
 973 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
 974 {
 975         struct sysctl_oid *oid, *child;
 976         struct sysctl_ctx_list *ctx_list;
 977
 978         ctx_list = device_get_sysctl_ctx(sc->dev);
 979         child = device_get_sysctl_tree(sc->dev);
 980         oid = SYSCTL_ADD_NODE(ctx_list,
 981             SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
 982             CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
 983         if (oid == NULL)
 984                 return (ENXIO);
 985
 986         /* Add registers */
 987         SYSCTL_ADD_PROC(ctx_list,
 988             SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
 989             CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 990             gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
 991         SYSCTL_ADD_PROC(ctx_list,
 992             SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
 993             CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 994             gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
 995
 996         return (0);
 997 }
 998
 999 static int
1000 gicv3_its_attach(device_t dev)
1001 {
1002         struct gicv3_its_softc *sc;
1003         int domain, err, i, rid;
1004         uint64_t phys;
1005         uint32_t ctlr, iidr;
1006
1007         sc = device_get_softc(dev);
1008
1009         sc->sc_dev_table_idx = -1;
1010         sc->sc_irq_length = gicv3_get_nirqs(dev);
1011         sc->sc_irq_base = GIC_FIRST_LPI;
1012         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
1013
1014         rid = 0;
1015         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1016             RF_ACTIVE);
1017         if (sc->sc_its_res == NULL) {
1018                 device_printf(dev, "Could not allocate memory\n");
1019                 return (ENXIO);
1020         }
1021
1022         phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
1023             GITS_TRANSLATER, PAGE_SIZE);
1024         sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
1025         vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
1026
1027         CPU_COPY(&all_cpus, &sc->sc_cpus);
1028         iidr = gic_its_read_4(sc, GITS_IIDR);
1029         for (i = 0; i < nitems(its_quirks); i++) {
1030                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
1031                         if (bootverbose) {
1032                                 device_printf(dev, "Applying %s\n",
1033                                     its_quirks[i].desc);
1034                         }
1035                         its_quirks[i].func(dev);
1036                         break;
1037                 }
1038         }
1039
1040         if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
1041                 sc->sc_ds = DOMAINSET_PREF(domain);
1042         } else {
1043                 sc->sc_ds = DOMAINSET_RR();
1044         }
1045
1046         /*
1047          * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
1048          * coming in via, for instance, a kexec/kboot style setup where a
1049          * previous kernel has configured then relinquished control.  Clear it
1050          * so that we can reconfigure GITS_BASER*.
1051          */
1052         ctlr = gic_its_read_4(sc, GITS_CTLR);
1053         if ((ctlr & GITS_CTLR_EN) != 0) {
1054                 ctlr &= ~GITS_CTLR_EN;
1055                 gic_its_write_4(sc, GITS_CTLR, ctlr);
1056         }
1057
1058         /* Allocate the private tables */
1059         err = gicv3_its_table_init(dev, sc);
1060         if (err != 0)
1061                 return (err);
1062
1063         /* Protects access to the device list */
1064         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
1065
1066         /* Protects access to the ITS command circular buffer. */
1067         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
1068
1069         /* Allocate the command circular buffer */
1070         gicv3_its_cmdq_init(sc);
1071
1072         /* Allocate the per-CPU collections */
1073         for (int cpu = 0; cpu <= mp_maxid; cpu++)
1074                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
1075                         sc->sc_its_cols[cpu] = malloc_domainset(
1076                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
1077                             DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
1078                             M_WAITOK | M_ZERO);
1079                 else
1080                         sc->sc_its_cols[cpu] = NULL;
1081
1082         /* Enable the ITS */
1083         gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
1084
1085         /* Create the LPI configuration table */
1086         gicv3_its_conftable_init(sc);
1087
1088         /* And the pending tebles */
1089         gicv3_its_pendtables_init(sc);
1090
1091         /* Enable LPIs on this CPU */
1092         its_init_cpu(dev, sc);
1093
1094         TAILQ_INIT(&sc->sc_its_dev_list);
1095         TAILQ_INIT(&sc->sc_free_irqs);
1096
1097         /*
1098          * Create the vmem object to allocate INTRNG IRQs from. We try to
1099          * use all IRQs not already used by the GICv3.
1100          * XXX: This assumes there are no other interrupt controllers in the
1101          * system.
1102          */
1103         sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
1104             gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
1105
1106         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
1107             M_GICV3_ITS, M_WAITOK | M_ZERO);
1108
1109         /* For GIC-500 install tracking sysctls. */
1110         if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
1111             GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
1112                 gicv3_its_init_sysctl(sc);
1113
1114         return (0);
1115 }
1116
1117 static int
1118 gicv3_its_detach(device_t dev)
1119 {
1120
1121         return (ENXIO);
1122 }
1123
1124 static void
1125 its_quirk_cavium_22375(device_t dev)
1126 {
1127         struct gicv3_its_softc *sc;
1128         int domain;
1129
1130         sc = device_get_softc(dev);
1131         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
1132
1133         /*
1134          * We need to limit which CPUs we send these interrupts to on
1135          * the original dual socket ThunderX as it is unable to
1136          * forward them between the two sockets.
1137          */
1138         if (bus_get_domain(dev, &domain) == 0) {
1139                 if (domain < MAXMEMDOM) {
1140                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1141                 } else {
1142                         CPU_ZERO(&sc->sc_cpus);
1143                 }
1144         }
1145 }
1146
1147 static void
1148 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1149 {
1150         struct gicv3_its_softc *sc;
1151         struct gicv3_its_irqsrc *girq;
1152         uint8_t *conf;
1153
1154         sc = device_get_softc(dev);
1155         girq = (struct gicv3_its_irqsrc *)isrc;
1156         conf = sc->sc_conf_base;
1157
1158         conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1159
1160         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1161                 /* Clean D-cache under command. */
1162                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1163         } else {
1164                 /* DSB inner shareable, store */
1165                 dsb(ishst);
1166         }
1167
1168         its_cmd_inv(dev, girq->gi_its_dev, girq);
1169 }
1170
1171 static void
1172 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1173 {
1174         struct gicv3_its_softc *sc;
1175         struct gicv3_its_irqsrc *girq;
1176         uint8_t *conf;
1177
1178         sc = device_get_softc(dev);
1179         girq = (struct gicv3_its_irqsrc *)isrc;
1180         conf = sc->sc_conf_base;
1181
1182         conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1183
1184         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1185                 /* Clean D-cache under command. */
1186                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1187         } else {
1188                 /* DSB inner shareable, store */
1189                 dsb(ishst);
1190         }
1191
1192         its_cmd_inv(dev, girq->gi_its_dev, girq);
1193 }
1194
1195 static int
1196 gicv3_its_intr(void *arg, uintptr_t irq)
1197 {
1198         struct gicv3_its_softc *sc = arg;
1199         struct gicv3_its_irqsrc *girq;
1200         struct trapframe *tf;
1201
1202         irq -= sc->sc_irq_base;
1203         girq = sc->sc_irqs[irq];
1204         if (girq == NULL)
1205                 panic("gicv3_its_intr: Invalid interrupt %ld",
1206                     irq + sc->sc_irq_base);
1207
1208         tf = curthread->td_intr_frame;
1209         intr_isrc_dispatch(&girq->gi_isrc, tf);
1210         return (FILTER_HANDLED);
1211 }
1212
1213 static void
1214 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1215 {
1216         struct gicv3_its_irqsrc *girq;
1217
1218         girq = (struct gicv3_its_irqsrc *)isrc;
1219         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1220 }
1221
1222 static void
1223 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1224 {
1225
1226 }
1227
1228 static void
1229 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1230 {
1231         struct gicv3_its_irqsrc *girq;
1232
1233         girq = (struct gicv3_its_irqsrc *)isrc;
1234         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1235 }
1236
1237 static int
1238 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1239 {
1240         struct gicv3_its_softc *sc;
1241
1242         sc = device_get_softc(dev);
1243         if (CPU_EMPTY(&isrc->isrc_cpu)) {
1244                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1245                     &sc->sc_cpus);
1246                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1247         }
1248
1249         return (0);
1250 }
1251
1252 static int
1253 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1254 {
1255         struct gicv3_its_irqsrc *girq;
1256
1257         gicv3_its_select_cpu(dev, isrc);
1258
1259         girq = (struct gicv3_its_irqsrc *)isrc;
1260         its_cmd_movi(dev, girq);
1261         return (0);
1262 }
1263
1264 static int
1265 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1266     struct intr_irqsrc **isrcp)
1267 {
1268
1269         /*
1270          * This should never happen, we only call this function to map
1271          * interrupts found before the controller driver is ready.
1272          */
1273         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1274 }
1275
1276 static int
1277 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1278     struct resource *res, struct intr_map_data *data)
1279 {
1280
1281         /* Bind the interrupt to a CPU */
1282         gicv3_its_bind_intr(dev, isrc);
1283
1284         return (0);
1285 }
1286
1287 #ifdef SMP
1288 static void
1289 gicv3_its_init_secondary(device_t dev)
1290 {
1291         struct gicv3_its_softc *sc;
1292
1293         sc = device_get_softc(dev);
1294
1295         /*
1296          * This is fatal as otherwise we may bind interrupts to this CPU.
1297          * We need a way to tell the interrupt framework to only bind to a
1298          * subset of given CPUs when it performs the shuffle.
1299          */
1300         if (its_init_cpu(dev, sc) != 0)
1301                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1302                     PCPU_GET(cpuid));
1303 }
1304 #endif
1305
1306 static uint32_t
1307 its_get_devid(device_t pci_dev)
1308 {
1309         uintptr_t id;
1310
1311         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1312                 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1313                     device_get_nameunit(pci_dev));
1314
1315         return (id);
1316 }
1317
1318 static struct its_dev *
1319 its_device_find(device_t dev, device_t child)
1320 {
1321         struct gicv3_its_softc *sc;
1322         struct its_dev *its_dev = NULL;
1323
1324         sc = device_get_softc(dev);
1325
1326         mtx_lock_spin(&sc->sc_its_dev_lock);
1327         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1328                 if (its_dev->pci_dev == child)
1329                         break;
1330         }
1331         mtx_unlock_spin(&sc->sc_its_dev_lock);
1332
1333         return (its_dev);
1334 }
1335
1336 static bool
1337 its_device_alloc(struct gicv3_its_softc *sc, int devid)
1338 {
1339         struct its_ptable *ptable;
1340         vm_offset_t l2_table;
1341         uint64_t *table;
1342         uint32_t index;
1343         bool shareable;
1344
1345         /* No device table */
1346         if (sc->sc_dev_table_idx < 0) {
1347                 if (devid >= (1 << sc->sc_devbits)) {
1348                         if (bootverbose) {
1349                                 device_printf(sc->dev,
1350                                     "%s: Device out of range for hardware "
1351                                     "(%x >= %x)\n", __func__, devid,
1352                                     1 << sc->sc_devbits);
1353                         }
1354                         return (false);
1355                 }
1356                 return (true);
1357         }
1358
1359         ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx];
1360         /* Check the devid is within the table limit */
1361         if (!ptable->ptab_indirect) {
1362                 if (devid >= ptable->ptab_l1_nidents) {
1363                         if (bootverbose) {
1364                                 device_printf(sc->dev,
1365                                     "%s: Device out of range for table "
1366                                     "(%x >= %x)\n", __func__, devid,
1367                                     ptable->ptab_l1_nidents);
1368                         }
1369                         return (false);
1370                 }
1371
1372                 return (true);
1373         }
1374
1375         /* Check the devid is within the allocated range */
1376         index = devid / ptable->ptab_l2_nidents;
1377         if (index >= ptable->ptab_l1_nidents) {
1378                 if (bootverbose) {
1379                         device_printf(sc->dev,
1380                             "%s: Index out of range for table (%x >= %x)\n",
1381                             __func__, index, ptable->ptab_l1_nidents);
1382                 }
1383                 return (false);
1384         }
1385
1386         table = (uint64_t *)ptable->ptab_vaddr;
1387         /* We have an second level table */
1388         if ((table[index] & GITS_BASER_VALID) != 0)
1389                 return (true);
1390
1391         shareable = true;
1392         if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS)
1393                 shareable = false;
1394
1395         l2_table = (vm_offset_t)contigmalloc_domainset(ptable->ptab_l2_size,
1396             M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
1397             ptable->ptab_page_size, 0);
1398
1399         if (!shareable)
1400                 cpu_dcache_wb_range(l2_table, ptable->ptab_l2_size);
1401
1402         table[index] = vtophys(l2_table) | GITS_BASER_VALID;
1403         if (!shareable)
1404                 cpu_dcache_wb_range((vm_offset_t)&table[index],
1405                     sizeof(table[index]));
1406
1407         dsb(sy);
1408         return (true);
1409 }
1410
1411 static struct its_dev *
1412 its_device_get(device_t dev, device_t child, u_int nvecs)
1413 {
1414         struct gicv3_its_softc *sc;
1415         struct its_dev *its_dev;
1416         vmem_addr_t irq_base;
1417         size_t esize;
1418
1419         sc = device_get_softc(dev);
1420
1421         its_dev = its_device_find(dev, child);
1422         if (its_dev != NULL)
1423                 return (its_dev);
1424
1425         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1426         if (its_dev == NULL)
1427                 return (NULL);
1428
1429         its_dev->pci_dev = child;
1430         its_dev->devid = its_get_devid(child);
1431
1432         its_dev->lpis.lpi_busy = 0;
1433         its_dev->lpis.lpi_num = nvecs;
1434         its_dev->lpis.lpi_free = nvecs;
1435
1436         if (!its_device_alloc(sc, its_dev->devid)) {
1437                 free(its_dev, M_GICV3_ITS);
1438                 return (NULL);
1439         }
1440
1441         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1442             &irq_base) != 0) {
1443                 free(its_dev, M_GICV3_ITS);
1444                 return (NULL);
1445         }
1446         its_dev->lpis.lpi_base = irq_base;
1447
1448         /* Get ITT entry size */
1449         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1450
1451         /*
1452          * Allocate ITT for this device.
1453          * PA has to be 256 B aligned. At least two entries for device.
1454          */
1455         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1456         its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
1457             M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1458             LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1459         if (its_dev->itt == 0) {
1460                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1461                 free(its_dev, M_GICV3_ITS);
1462                 return (NULL);
1463         }
1464
1465         /* Make sure device sees zeroed ITT. */
1466         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1467                 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1468
1469         mtx_lock_spin(&sc->sc_its_dev_lock);
1470         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1471         mtx_unlock_spin(&sc->sc_its_dev_lock);
1472
1473         /* Map device to its ITT */
1474         its_cmd_mapd(dev, its_dev, 1);
1475
1476         return (its_dev);
1477 }
1478
1479 static void
1480 its_device_release(device_t dev, struct its_dev *its_dev)
1481 {
1482         struct gicv3_its_softc *sc;
1483
1484         KASSERT(its_dev->lpis.lpi_busy == 0,
1485             ("its_device_release: Trying to release an inuse ITS device"));
1486
1487         /* Unmap device in ITS */
1488         its_cmd_mapd(dev, its_dev, 0);
1489
1490         sc = device_get_softc(dev);
1491
1492         /* Remove the device from the list of devices */
1493         mtx_lock_spin(&sc->sc_its_dev_lock);
1494         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1495         mtx_unlock_spin(&sc->sc_its_dev_lock);
1496
1497         /* Free ITT */
1498         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1499         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1500
1501         /* Free the IRQ allocation */
1502         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1503             its_dev->lpis.lpi_num);
1504
1505         free(its_dev, M_GICV3_ITS);
1506 }
1507
1508 static struct gicv3_its_irqsrc *
1509 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1510 {
1511         struct gicv3_its_irqsrc *girq = NULL;
1512
1513         KASSERT(sc->sc_irqs[irq] == NULL,
1514             ("%s: Interrupt %u already allocated", __func__, irq));
1515         mtx_lock_spin(&sc->sc_its_dev_lock);
1516         if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1517                 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1518                 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1519         }
1520         mtx_unlock_spin(&sc->sc_its_dev_lock);
1521         if (girq == NULL) {
1522                 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1523                     M_NOWAIT | M_ZERO);
1524                 if (girq == NULL)
1525                         return (NULL);
1526                 girq->gi_id = -1;
1527                 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1528                     "%s,%u", device_get_nameunit(dev), irq) != 0) {
1529                         free(girq, M_GICV3_ITS);
1530                         return (NULL);
1531                 }
1532         }
1533         girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1534         sc->sc_irqs[irq] = girq;
1535
1536         return (girq);
1537 }
1538
1539 static void
1540 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1541     struct gicv3_its_irqsrc *girq)
1542 {
1543         u_int irq;
1544
1545         mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1546
1547         irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1548         sc->sc_irqs[irq] = NULL;
1549
1550         girq->gi_id = -1;
1551         girq->gi_its_dev = NULL;
1552         TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1553 }
1554
1555 static int
1556 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1557     device_t *pic, struct intr_irqsrc **srcs)
1558 {
1559         struct gicv3_its_softc *sc;
1560         struct gicv3_its_irqsrc *girq;
1561         struct its_dev *its_dev;
1562         u_int irq;
1563         int i;
1564
1565         its_dev = its_device_get(dev, child, count);
1566         if (its_dev == NULL)
1567                 return (ENXIO);
1568
1569         KASSERT(its_dev->lpis.lpi_free >= count,
1570             ("gicv3_its_alloc_msi: No free LPIs"));
1571         sc = device_get_softc(dev);
1572         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1573             its_dev->lpis.lpi_free;
1574
1575         /* Allocate the irqsrc for each MSI */
1576         for (i = 0; i < count; i++, irq++) {
1577                 its_dev->lpis.lpi_free--;
1578                 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1579                     sc, irq);
1580                 if (srcs[i] == NULL)
1581                         break;
1582         }
1583
1584         /* The allocation failed, release them */
1585         if (i != count) {
1586                 mtx_lock_spin(&sc->sc_its_dev_lock);
1587                 for (i = 0; i < count; i++) {
1588                         girq = (struct gicv3_its_irqsrc *)srcs[i];
1589                         if (girq == NULL)
1590                                 break;
1591                         gicv3_its_release_irqsrc(sc, girq);
1592                         srcs[i] = NULL;
1593                 }
1594                 mtx_unlock_spin(&sc->sc_its_dev_lock);
1595                 return (ENXIO);
1596         }
1597
1598         /* Finish the allocation now we have all MSI irqsrcs */
1599         for (i = 0; i < count; i++) {
1600                 girq = (struct gicv3_its_irqsrc *)srcs[i];
1601                 girq->gi_id = i;
1602                 girq->gi_its_dev = its_dev;
1603
1604                 /* Map the message to the given IRQ */
1605                 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1606                 its_cmd_mapti(dev, girq);
1607         }
1608         its_dev->lpis.lpi_busy += count;
1609         *pic = dev;
1610
1611         return (0);
1612 }
1613
1614 static int
1615 gicv3_its_release_msi(device_t dev, device_t child, int count,
1616     struct intr_irqsrc **isrc)
1617 {
1618         struct gicv3_its_softc *sc;
1619         struct gicv3_its_irqsrc *girq;
1620         struct its_dev *its_dev;
1621         int i;
1622
1623         its_dev = its_device_find(dev, child);
1624
1625         KASSERT(its_dev != NULL,
1626             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1627              "no ITS device"));
1628         KASSERT(its_dev->lpis.lpi_busy >= count,
1629             ("gicv3_its_release_msi: Releasing more interrupts than "
1630              "were allocated: releasing %d, allocated %d", count,
1631              its_dev->lpis.lpi_busy));
1632
1633         sc = device_get_softc(dev);
1634         mtx_lock_spin(&sc->sc_its_dev_lock);
1635         for (i = 0; i < count; i++) {
1636                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1637                 gicv3_its_release_irqsrc(sc, girq);
1638         }
1639         mtx_unlock_spin(&sc->sc_its_dev_lock);
1640         its_dev->lpis.lpi_busy -= count;
1641
1642         if (its_dev->lpis.lpi_busy == 0)
1643                 its_device_release(dev, its_dev);
1644
1645         return (0);
1646 }
1647
1648 static int
1649 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1650     struct intr_irqsrc **isrcp)
1651 {
1652         struct gicv3_its_softc *sc;
1653         struct gicv3_its_irqsrc *girq;
1654         struct its_dev *its_dev;
1655         u_int nvecs, irq;
1656
1657         nvecs = pci_msix_count(child);
1658         its_dev = its_device_get(dev, child, nvecs);
1659         if (its_dev == NULL)
1660                 return (ENXIO);
1661
1662         KASSERT(its_dev->lpis.lpi_free > 0,
1663             ("gicv3_its_alloc_msix: No free LPIs"));
1664         sc = device_get_softc(dev);
1665         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1666             its_dev->lpis.lpi_free;
1667
1668         girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1669         if (girq == NULL)
1670                 return (ENXIO);
1671         girq->gi_id = its_dev->lpis.lpi_busy;
1672         girq->gi_its_dev = its_dev;
1673
1674         its_dev->lpis.lpi_free--;
1675         its_dev->lpis.lpi_busy++;
1676
1677         /* Map the message to the given IRQ */
1678         gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1679         its_cmd_mapti(dev, girq);
1680
1681         *pic = dev;
1682         *isrcp = (struct intr_irqsrc *)girq;
1683
1684         return (0);
1685 }
1686
1687 static int
1688 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1689 {
1690         struct gicv3_its_softc *sc;
1691         struct gicv3_its_irqsrc *girq;
1692         struct its_dev *its_dev;
1693
1694         its_dev = its_device_find(dev, child);
1695
1696         KASSERT(its_dev != NULL,
1697             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1698              "no ITS device"));
1699         KASSERT(its_dev->lpis.lpi_busy > 0,
1700             ("gicv3_its_release_msix: Releasing more interrupts than "
1701              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1702
1703         sc = device_get_softc(dev);
1704         girq = (struct gicv3_its_irqsrc *)isrc;
1705         mtx_lock_spin(&sc->sc_its_dev_lock);
1706         gicv3_its_release_irqsrc(sc, girq);
1707         mtx_unlock_spin(&sc->sc_its_dev_lock);
1708         its_dev->lpis.lpi_busy--;
1709
1710         if (its_dev->lpis.lpi_busy == 0)
1711                 its_device_release(dev, its_dev);
1712
1713         return (0);
1714 }
1715
1716 static int
1717 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1718     uint64_t *addr, uint32_t *data)
1719 {
1720         struct gicv3_its_softc *sc;
1721         struct gicv3_its_irqsrc *girq;
1722
1723         sc = device_get_softc(dev);
1724         girq = (struct gicv3_its_irqsrc *)isrc;
1725
1726         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1727         *data = girq->gi_id;
1728
1729         return (0);
1730 }
1731
1732 #ifdef IOMMU
1733 static int
1734 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1735 {
1736         struct gicv3_its_softc *sc;
1737         struct iommu_ctx *ctx;
1738         int error;
1739
1740         sc = device_get_softc(dev);
1741         ctx = iommu_get_dev_ctx(child);
1742         if (ctx == NULL)
1743                 return (ENXIO);
1744         /* Map the page containing the GITS_TRANSLATER register. */
1745         error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1746             IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1747         *domain = iommu_get_ctx_domain(ctx);
1748
1749         return (error);
1750 }
1751
1752 static void
1753 gicv3_iommu_deinit(device_t dev, device_t child)
1754 {
1755         struct iommu_ctx *ctx;
1756
1757         ctx = iommu_get_dev_ctx(child);
1758         if (ctx == NULL)
1759                 return;
1760
1761         iommu_unmap_msi(ctx);
1762 }
1763 #endif
1764
1765 /*
1766  * Commands handling.
1767  */
1768
1769 static __inline void
1770 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1771 {
1772         /* Command field: DW0 [7:0] */
1773         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1774         cmd->cmd_dword[0] |= htole64(cmd_type);
1775 }
1776
1777 static __inline void
1778 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1779 {
1780         /* Device ID field: DW0 [63:32] */
1781         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1782         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1783 }
1784
1785 static __inline void
1786 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1787 {
1788         /* Size field: DW1 [4:0] */
1789         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1790         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1791 }
1792
1793 static __inline void
1794 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1795 {
1796         /* ID field: DW1 [31:0] */
1797         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1798         cmd->cmd_dword[1] |= htole64(id);
1799 }
1800
1801 static __inline void
1802 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1803 {
1804         /* Physical ID field: DW1 [63:32] */
1805         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1806         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1807 }
1808
1809 static __inline void
1810 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1811 {
1812         /* Collection field: DW2 [16:0] */
1813         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1814         cmd->cmd_dword[2] |= htole64(col_id);
1815 }
1816
1817 static __inline void
1818 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1819 {
1820         /* Target Address field: DW2 [47:16] */
1821         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1822         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1823 }
1824
1825 static __inline void
1826 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1827 {
1828         /* ITT Address field: DW2 [47:8] */
1829         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1830         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1831 }
1832
1833 static __inline void
1834 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1835 {
1836         /* Valid field: DW2 [63] */
1837         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1838         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1839 }
1840
1841 static inline bool
1842 its_cmd_queue_full(struct gicv3_its_softc *sc)
1843 {
1844         size_t read_idx, next_write_idx;
1845
1846         /* Get the index of the next command */
1847         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1848             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1849         /* And the index of the current command being read */
1850         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1851
1852         /*
1853          * The queue is full when the write offset points
1854          * at the command before the current read offset.
1855          */
1856         return (next_write_idx == read_idx);
1857 }
1858
1859 static inline void
1860 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1861 {
1862
1863         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1864                 /* Clean D-cache under command. */
1865                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1866         } else {
1867                 /* DSB inner shareable, store */
1868                 dsb(ishst);
1869         }
1870
1871 }
1872
1873 static inline uint64_t
1874 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1875 {
1876         uint64_t off;
1877
1878         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1879
1880         return (off);
1881 }
1882
1883 static void
1884 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1885     struct its_cmd *cmd_last)
1886 {
1887         struct gicv3_its_softc *sc;
1888         uint64_t first, last, read;
1889         size_t us_left;
1890
1891         sc = device_get_softc(dev);
1892
1893         /*
1894          * XXX ARM64TODO: This is obviously a significant delay.
1895          * The reason for that is that currently the time frames for
1896          * the command to complete are not known.
1897          */
1898         us_left = 1000000;
1899
1900         first = its_cmd_cwriter_offset(sc, cmd_first);
1901         last = its_cmd_cwriter_offset(sc, cmd_last);
1902
1903         for (;;) {
1904                 read = gic_its_read_8(sc, GITS_CREADR);
1905                 if (first < last) {
1906                         if (read < first || read >= last)
1907                                 break;
1908                 } else if (read < first && read >= last)
1909                         break;
1910
1911                 if (us_left-- == 0) {
1912                         /* This means timeout */
1913                         device_printf(dev,
1914                             "Timeout while waiting for CMD completion.\n");
1915                         return;
1916                 }
1917                 DELAY(1);
1918         }
1919 }
1920
1921 static struct its_cmd *
1922 its_cmd_alloc_locked(device_t dev)
1923 {
1924         struct gicv3_its_softc *sc;
1925         struct its_cmd *cmd;
1926         size_t us_left;
1927
1928         sc = device_get_softc(dev);
1929
1930         /*
1931          * XXX ARM64TODO: This is obviously a significant delay.
1932          * The reason for that is that currently the time frames for
1933          * the command to complete (and therefore free the descriptor)
1934          * are not known.
1935          */
1936         us_left = 1000000;
1937
1938         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1939         while (its_cmd_queue_full(sc)) {
1940                 if (us_left-- == 0) {
1941                         /* Timeout while waiting for free command */
1942                         device_printf(dev,
1943                             "Timeout while waiting for free command\n");
1944                         return (NULL);
1945                 }
1946                 DELAY(1);
1947         }
1948
1949         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1950         sc->sc_its_cmd_next_idx++;
1951         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1952
1953         return (cmd);
1954 }
1955
1956 static uint64_t
1957 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1958 {
1959         uint64_t target;
1960         uint8_t cmd_type;
1961         u_int size;
1962
1963         cmd_type = desc->cmd_type;
1964         target = ITS_TARGET_NONE;
1965
1966         switch (cmd_type) {
1967         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1968                 target = desc->cmd_desc_movi.col->col_target;
1969                 cmd_format_command(cmd, ITS_CMD_MOVI);
1970                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1971                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1972                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1973                 break;
1974         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1975                 target = desc->cmd_desc_sync.col->col_target;
1976                 cmd_format_command(cmd, ITS_CMD_SYNC);
1977                 cmd_format_target(cmd, target);
1978                 break;
1979         case ITS_CMD_MAPD:      /* Assign ITT to device */
1980                 cmd_format_command(cmd, ITS_CMD_MAPD);
1981                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1982                 /*
1983                  * Size describes number of bits to encode interrupt IDs
1984                  * supported by the device minus one.
1985                  * When V (valid) bit is zero, this field should be written
1986                  * as zero.
1987                  */
1988                 if (desc->cmd_desc_mapd.valid != 0) {
1989                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1990                         size = MAX(1, size) - 1;
1991                 } else
1992                         size = 0;
1993
1994                 cmd_format_size(cmd, size);
1995                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1996                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1997                 break;
1998         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1999                 target = desc->cmd_desc_mapc.col->col_target;
2000                 cmd_format_command(cmd, ITS_CMD_MAPC);
2001                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
2002                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
2003                 cmd_format_target(cmd, target);
2004                 break;
2005         case ITS_CMD_MAPTI:
2006                 target = desc->cmd_desc_mapvi.col->col_target;
2007                 cmd_format_command(cmd, ITS_CMD_MAPTI);
2008                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
2009                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
2010                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
2011                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
2012                 break;
2013         case ITS_CMD_MAPI:
2014                 target = desc->cmd_desc_mapi.col->col_target;
2015                 cmd_format_command(cmd, ITS_CMD_MAPI);
2016                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
2017                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
2018                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
2019                 break;
2020         case ITS_CMD_INV:
2021                 target = desc->cmd_desc_inv.col->col_target;
2022                 cmd_format_command(cmd, ITS_CMD_INV);
2023                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
2024                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
2025                 break;
2026         case ITS_CMD_INVALL:
2027                 cmd_format_command(cmd, ITS_CMD_INVALL);
2028                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
2029                 break;
2030         default:
2031                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
2032         }
2033
2034         return (target);
2035 }
2036
2037 static int
2038 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
2039 {
2040         struct gicv3_its_softc *sc;
2041         struct its_cmd *cmd, *cmd_sync, *cmd_write;
2042         struct its_col col_sync;
2043         struct its_cmd_desc desc_sync;
2044         uint64_t target, cwriter;
2045
2046         sc = device_get_softc(dev);
2047         mtx_lock_spin(&sc->sc_its_cmd_lock);
2048         cmd = its_cmd_alloc_locked(dev);
2049         if (cmd == NULL) {
2050                 device_printf(dev, "could not allocate ITS command\n");
2051                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2052                 return (EBUSY);
2053         }
2054
2055         target = its_cmd_prepare(cmd, desc);
2056         its_cmd_sync(sc, cmd);
2057
2058         if (target != ITS_TARGET_NONE) {
2059                 cmd_sync = its_cmd_alloc_locked(dev);
2060                 if (cmd_sync != NULL) {
2061                         desc_sync.cmd_type = ITS_CMD_SYNC;
2062                         col_sync.col_target = target;
2063                         desc_sync.cmd_desc_sync.col = &col_sync;
2064                         its_cmd_prepare(cmd_sync, &desc_sync);
2065                         its_cmd_sync(sc, cmd_sync);
2066                 }
2067         }
2068
2069         /* Update GITS_CWRITER */
2070         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
2071         gic_its_write_8(sc, GITS_CWRITER, cwriter);
2072         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2073         mtx_unlock_spin(&sc->sc_its_cmd_lock);
2074
2075         its_cmd_wait_completion(dev, cmd, cmd_write);
2076
2077         return (0);
2078 }
2079
2080 /* Handlers to send commands */
2081 static void
2082 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
2083 {
2084         struct gicv3_its_softc *sc;
2085         struct its_cmd_desc desc;
2086         struct its_col *col;
2087
2088         sc = device_get_softc(dev);
2089         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2090
2091         desc.cmd_type = ITS_CMD_MOVI;
2092         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
2093         desc.cmd_desc_movi.col = col;
2094         desc.cmd_desc_movi.id = girq->gi_id;
2095
2096         its_cmd_send(dev, &desc);
2097 }
2098
2099 static void
2100 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
2101 {
2102         struct its_cmd_desc desc;
2103
2104         desc.cmd_type = ITS_CMD_MAPC;
2105         desc.cmd_desc_mapc.col = col;
2106         /*
2107          * Valid bit set - map the collection.
2108          * Valid bit cleared - unmap the collection.
2109          */
2110         desc.cmd_desc_mapc.valid = valid;
2111
2112         its_cmd_send(dev, &desc);
2113 }
2114
2115 static void
2116 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
2117 {
2118         struct gicv3_its_softc *sc;
2119         struct its_cmd_desc desc;
2120         struct its_col *col;
2121         u_int col_id;
2122
2123         sc = device_get_softc(dev);
2124
2125         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
2126         col = sc->sc_its_cols[col_id];
2127
2128         desc.cmd_type = ITS_CMD_MAPTI;
2129         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
2130         desc.cmd_desc_mapvi.col = col;
2131         /* The EventID sent to the device */
2132         desc.cmd_desc_mapvi.id = girq->gi_id;
2133         /* The physical interrupt presented to softeware */
2134         desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
2135
2136         its_cmd_send(dev, &desc);
2137 }
2138
2139 static void
2140 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
2141 {
2142         struct its_cmd_desc desc;
2143
2144         desc.cmd_type = ITS_CMD_MAPD;
2145         desc.cmd_desc_mapd.its_dev = its_dev;
2146         desc.cmd_desc_mapd.valid = valid;
2147
2148         its_cmd_send(dev, &desc);
2149 }
2150
2151 static void
2152 its_cmd_inv(device_t dev, struct its_dev *its_dev,
2153     struct gicv3_its_irqsrc *girq)
2154 {
2155         struct gicv3_its_softc *sc;
2156         struct its_cmd_desc desc;
2157         struct its_col *col;
2158
2159         sc = device_get_softc(dev);
2160         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2161
2162         desc.cmd_type = ITS_CMD_INV;
2163         /* The EventID sent to the device */
2164         desc.cmd_desc_inv.pid = girq->gi_id;
2165         desc.cmd_desc_inv.its_dev = its_dev;
2166         desc.cmd_desc_inv.col = col;
2167
2168         its_cmd_send(dev, &desc);
2169 }
2170
2171 static void
2172 its_cmd_invall(device_t dev, struct its_col *col)
2173 {
2174         struct its_cmd_desc desc;
2175
2176         desc.cmd_type = ITS_CMD_INVALL;
2177         desc.cmd_desc_invall.col = col;
2178
2179         its_cmd_send(dev, &desc);
2180 }
2181
2182 #ifdef FDT
2183 static device_probe_t gicv3_its_fdt_probe;
2184 static device_attach_t gicv3_its_fdt_attach;
2185
2186 static device_method_t gicv3_its_fdt_methods[] = {
2187         /* Device interface */
2188         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
2189         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
2190
2191         /* End */
2192         DEVMETHOD_END
2193 };
2194
2195 #define its_baseclasses its_fdt_baseclasses
2196 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
2197     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2198 #undef its_baseclasses
2199
2200 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
2201     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2202
2203 static int
2204 gicv3_its_fdt_probe(device_t dev)
2205 {
2206
2207         if (!ofw_bus_status_okay(dev))
2208                 return (ENXIO);
2209
2210         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
2211                 return (ENXIO);
2212
2213         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2214         return (BUS_PROBE_DEFAULT);
2215 }
2216
2217 static int
2218 gicv3_its_fdt_attach(device_t dev)
2219 {
2220         struct gicv3_its_softc *sc;
2221         phandle_t xref;
2222         int err;
2223
2224         sc = device_get_softc(dev);
2225         sc->dev = dev;
2226         err = gicv3_its_attach(dev);
2227         if (err != 0)
2228                 return (err);
2229
2230         /* Register this device as a interrupt controller */
2231         xref = OF_xref_from_node(ofw_bus_get_node(dev));
2232         sc->sc_pic = intr_pic_register(dev, xref);
2233         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2234             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2235         if (err != 0) {
2236                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2237                 return (err);
2238         }
2239
2240         /* Register this device to handle MSI interrupts */
2241         err = intr_msi_register(dev, xref);
2242         if (err != 0) {
2243                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2244                 return (err);
2245         }
2246
2247         return (0);
2248 }
2249 #endif
2250
2251 #ifdef DEV_ACPI
2252 static device_probe_t gicv3_its_acpi_probe;
2253 static device_attach_t gicv3_its_acpi_attach;
2254
2255 static device_method_t gicv3_its_acpi_methods[] = {
2256         /* Device interface */
2257         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
2258         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
2259
2260         /* End */
2261         DEVMETHOD_END
2262 };
2263
2264 #define its_baseclasses its_acpi_baseclasses
2265 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2266     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2267 #undef its_baseclasses
2268
2269 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2270     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2271
2272 static int
2273 gicv3_its_acpi_probe(device_t dev)
2274 {
2275
2276         if (gic_get_bus(dev) != GIC_BUS_ACPI)
2277                 return (EINVAL);
2278
2279         if (gic_get_hw_rev(dev) < 3)
2280                 return (EINVAL);
2281
2282         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2283         return (BUS_PROBE_DEFAULT);
2284 }
2285
2286 static int
2287 gicv3_its_acpi_attach(device_t dev)
2288 {
2289         struct gicv3_its_softc *sc;
2290         struct gic_v3_devinfo *di;
2291         int err;
2292
2293         sc = device_get_softc(dev);
2294         sc->dev = dev;
2295         err = gicv3_its_attach(dev);
2296         if (err != 0)
2297                 return (err);
2298
2299         di = device_get_ivars(dev);
2300         sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2301         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2302             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2303         if (err != 0) {
2304                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2305                 return (err);
2306         }
2307
2308         /* Register this device to handle MSI interrupts */
2309         err = intr_msi_register(dev, di->msi_xref);
2310         if (err != 0) {
2311                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2312                 return (err);
2313         }
2314
2315         return (0);
2316 }
2317 #endif