sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  * Copyright (c) 2023 Arm Ltd
   4  *
   5  * This software was developed by Andrew Turner under
   6  * the sponsorship of the FreeBSD Foundation.
   7  *
   8  * This software was developed by Semihalf under
   9  * the sponsorship of the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include "opt_acpi.h"
  34 #include "opt_platform.h"
  35 #include "opt_iommu.h"
  36
  37 #include <sys/cdefs.h>
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/bus.h>
  41 #include <sys/cpuset.h>
  42 #include <sys/domainset.h>
  43 #include <sys/endian.h>
  44 #include <sys/kernel.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mutex.h>
  49 #include <sys/proc.h>
  50 #include <sys/taskqueue.h>
  51 #include <sys/tree.h>
  52 #include <sys/queue.h>
  53 #include <sys/rman.h>
  54 #include <sys/sbuf.h>
  55 #include <sys/smp.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/vmem.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #include <vm/vm_page.h>
  62
  63 #include <machine/bus.h>
  64 #include <machine/intr.h>
  65
  66 #include <arm/arm/gic_common.h>
  67 #include <arm64/arm64/gic_v3_reg.h>
  68 #include <arm64/arm64/gic_v3_var.h>
  69
  70 #ifdef FDT
  71 #include <dev/ofw/openfirm.h>
  72 #include <dev/ofw/ofw_bus.h>
  73 #include <dev/ofw/ofw_bus_subr.h>
  74 #endif
  75 #include <dev/pci/pcireg.h>
  76 #include <dev/pci/pcivar.h>
  77
  78 #ifdef IOMMU
  79 #include <dev/iommu/iommu.h>
  80 #include <dev/iommu/iommu_gas.h>
  81 #endif
  82
  83 #include "pcib_if.h"
  84 #include "pic_if.h"
  85 #include "msi_if.h"
  86
  87 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  88     "ARM GICv3 Interrupt Translation Service");
  89
  90 #define LPI_NIRQS               (64 * 1024)
  91
  92 /* The size and alignment of the command circular buffer */
  93 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  94 #define ITS_CMDQ_ALIGN          (64 * 1024)
  95
  96 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  97 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  98 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  99
 100 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
 101 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
 102 #define LPI_PENDTAB_ALIGN       (64 * 1024)
 103 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 104
 105 #define LPI_INT_TRANS_TAB_ALIGN 256
 106 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
 107
 108 /* ITS commands encoding */
 109 #define ITS_CMD_MOVI            (0x01)
 110 #define ITS_CMD_SYNC            (0x05)
 111 #define ITS_CMD_MAPD            (0x08)
 112 #define ITS_CMD_MAPC            (0x09)
 113 #define ITS_CMD_MAPTI           (0x0a)
 114 #define ITS_CMD_MAPI            (0x0b)
 115 #define ITS_CMD_INV             (0x0c)
 116 #define ITS_CMD_INVALL          (0x0d)
 117 /* Command */
 118 #define CMD_COMMAND_MASK        (0xFFUL)
 119 /* PCI device ID */
 120 #define CMD_DEVID_SHIFT         (32)
 121 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 122 /* Size of IRQ ID bitfield */
 123 #define CMD_SIZE_MASK           (0xFFUL)
 124 /* Virtual LPI ID */
 125 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 126 /* Physical LPI ID */
 127 #define CMD_PID_SHIFT           (32)
 128 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 129 /* Collection */
 130 #define CMD_COL_MASK            (0xFFFFUL)
 131 /* Target (CPU or Re-Distributor) */
 132 #define CMD_TARGET_SHIFT        (16)
 133 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 134 /* Interrupt Translation Table address */
 135 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 136 /* Valid command bit */
 137 #define CMD_VALID_SHIFT         (63)
 138 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 139
 140 #define ITS_TARGET_NONE         0xFBADBEEF
 141
 142 /* LPI chunk owned by ITS device */
 143 struct lpi_chunk {
 144         u_int   lpi_base;
 145         u_int   lpi_free;       /* First free LPI in set */
 146         u_int   lpi_num;        /* Total number of LPIs in chunk */
 147         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 148 };
 149
 150 /* ITS device */
 151 struct its_dev {
 152         TAILQ_ENTRY(its_dev)    entry;
 153         /* PCI device */
 154         device_t                pci_dev;
 155         /* Device ID (i.e. PCI device ID) */
 156         uint32_t                devid;
 157         /* List of assigned LPIs */
 158         struct lpi_chunk        lpis;
 159         /* Virtual address of ITT */
 160         vm_offset_t             itt;
 161         size_t                  itt_size;
 162 };
 163
 164 /*
 165  * ITS command descriptor.
 166  * Idea for command description passing taken from Linux.
 167  */
 168 struct its_cmd_desc {
 169         uint8_t cmd_type;
 170
 171         union {
 172                 struct {
 173                         struct its_dev *its_dev;
 174                         struct its_col *col;
 175                         uint32_t id;
 176                 } cmd_desc_movi;
 177
 178                 struct {
 179                         struct its_col *col;
 180                 } cmd_desc_sync;
 181
 182                 struct {
 183                         struct its_col *col;
 184                         uint8_t valid;
 185                 } cmd_desc_mapc;
 186
 187                 struct {
 188                         struct its_dev *its_dev;
 189                         struct its_col *col;
 190                         uint32_t pid;
 191                         uint32_t id;
 192                 } cmd_desc_mapvi;
 193
 194                 struct {
 195                         struct its_dev *its_dev;
 196                         struct its_col *col;
 197                         uint32_t pid;
 198                 } cmd_desc_mapi;
 199
 200                 struct {
 201                         struct its_dev *its_dev;
 202                         uint8_t valid;
 203                 } cmd_desc_mapd;
 204
 205                 struct {
 206                         struct its_dev *its_dev;
 207                         struct its_col *col;
 208                         uint32_t pid;
 209                 } cmd_desc_inv;
 210
 211                 struct {
 212                         struct its_col *col;
 213                 } cmd_desc_invall;
 214         };
 215 };
 216
 217 /* ITS command. Each command is 32 bytes long */
 218 struct its_cmd {
 219         uint64_t        cmd_dword[4];   /* ITS command double word */
 220 };
 221
 222 /* An ITS private table */
 223 struct its_ptable {
 224         vm_offset_t     ptab_vaddr;
 225         /* Size of the L1 and L2 tables */
 226         size_t          ptab_l1_size;
 227         size_t          ptab_l2_size;
 228         /* Number of L1 and L2 entries */
 229         int             ptab_l1_nidents;
 230         int             ptab_l2_nidents;
 231
 232         int             ptab_page_size;
 233         int             ptab_share;
 234         bool            ptab_indirect;
 235 };
 236
 237 /* ITS collection description. */
 238 struct its_col {
 239         uint64_t        col_target;     /* Target Re-Distributor */
 240         uint64_t        col_id;         /* Collection ID */
 241 };
 242
 243 struct gicv3_its_irqsrc {
 244         struct intr_irqsrc      gi_isrc;
 245         u_int                   gi_id;
 246         u_int                   gi_lpi;
 247         struct its_dev          *gi_its_dev;
 248         TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
 249 };
 250
 251 struct gicv3_its_softc {
 252         device_t        dev;
 253         struct intr_pic *sc_pic;
 254         struct resource *sc_its_res;
 255
 256         cpuset_t        sc_cpus;
 257         struct domainset *sc_ds;
 258         u_int           gic_irq_cpu;
 259         int             sc_devbits;
 260         int             sc_dev_table_idx;
 261
 262         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 263         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 264
 265         /*
 266          * TODO: We should get these from the parent as we only want a
 267          * single copy of each across the interrupt controller.
 268          */
 269         uint8_t         *sc_conf_base;
 270         vm_offset_t sc_pend_base[MAXCPU];
 271
 272         /* Command handling */
 273         struct mtx sc_its_cmd_lock;
 274         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 275         size_t sc_its_cmd_next_idx;
 276
 277         vmem_t *sc_irq_alloc;
 278         struct gicv3_its_irqsrc **sc_irqs;
 279         u_int   sc_irq_base;
 280         u_int   sc_irq_length;
 281         u_int   sc_irq_count;
 282
 283         struct mtx sc_its_dev_lock;
 284         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 285         TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
 286
 287 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 288 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 289 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 290         u_int sc_its_flags;
 291         bool    trace_enable;
 292         vm_page_t ma; /* fake msi page */
 293 };
 294
 295 static void *conf_base;
 296
 297 typedef void (its_quirk_func_t)(device_t);
 298 static its_quirk_func_t its_quirk_cavium_22375;
 299
 300 static const struct {
 301         const char *desc;
 302         uint32_t iidr;
 303         uint32_t iidr_mask;
 304         its_quirk_func_t *func;
 305 } its_quirks[] = {
 306         {
 307                 /* Cavium ThunderX Pass 1.x */
 308                 .desc = "Cavium ThunderX errata: 22375, 24313",
 309                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 310                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 311                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 312                 .func = its_quirk_cavium_22375,
 313         },
 314 };
 315
 316 #define gic_its_read_4(sc, reg)                 \
 317     bus_read_4((sc)->sc_its_res, (reg))
 318 #define gic_its_read_8(sc, reg)                 \
 319     bus_read_8((sc)->sc_its_res, (reg))
 320
 321 #define gic_its_write_4(sc, reg, val)           \
 322     bus_write_4((sc)->sc_its_res, (reg), (val))
 323 #define gic_its_write_8(sc, reg, val)           \
 324     bus_write_8((sc)->sc_its_res, (reg), (val))
 325
 326 static device_attach_t gicv3_its_attach;
 327 static device_detach_t gicv3_its_detach;
 328
 329 static pic_disable_intr_t gicv3_its_disable_intr;
 330 static pic_enable_intr_t gicv3_its_enable_intr;
 331 static pic_map_intr_t gicv3_its_map_intr;
 332 static pic_setup_intr_t gicv3_its_setup_intr;
 333 static pic_post_filter_t gicv3_its_post_filter;
 334 static pic_post_ithread_t gicv3_its_post_ithread;
 335 static pic_pre_ithread_t gicv3_its_pre_ithread;
 336 static pic_bind_intr_t gicv3_its_bind_intr;
 337 #ifdef SMP
 338 static pic_init_secondary_t gicv3_its_init_secondary;
 339 #endif
 340 static msi_alloc_msi_t gicv3_its_alloc_msi;
 341 static msi_release_msi_t gicv3_its_release_msi;
 342 static msi_alloc_msix_t gicv3_its_alloc_msix;
 343 static msi_release_msix_t gicv3_its_release_msix;
 344 static msi_map_msi_t gicv3_its_map_msi;
 345 #ifdef IOMMU
 346 static msi_iommu_init_t gicv3_iommu_init;
 347 static msi_iommu_deinit_t gicv3_iommu_deinit;
 348 #endif
 349
 350 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 351 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 352 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 353 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 354 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 355 static void its_cmd_invall(device_t, struct its_col *);
 356
 357 static device_method_t gicv3_its_methods[] = {
 358         /* Device interface */
 359         DEVMETHOD(device_detach,        gicv3_its_detach),
 360
 361         /* Interrupt controller interface */
 362         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 363         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 364         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 365         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 366         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 367         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 368         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 369 #ifdef SMP
 370         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 371         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 372 #endif
 373
 374         /* MSI/MSI-X */
 375         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 376         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 377         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 378         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 379         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 380 #ifdef IOMMU
 381         DEVMETHOD(msi_iommu_init,       gicv3_iommu_init),
 382         DEVMETHOD(msi_iommu_deinit,     gicv3_iommu_deinit),
 383 #endif
 384
 385         /* End */
 386         DEVMETHOD_END
 387 };
 388
 389 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 390     sizeof(struct gicv3_its_softc));
 391
 392 static void
 393 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 394 {
 395         vm_paddr_t cmd_paddr;
 396         uint64_t reg, tmp;
 397
 398         /* Set up the command circular buffer */
 399         sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
 400             sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
 401             0);
 402         sc->sc_its_cmd_next_idx = 0;
 403
 404         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 405
 406         /* Set the base of the command buffer */
 407         reg = GITS_CBASER_VALID |
 408             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 409             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 410             (ITS_CMDQ_SIZE / 4096 - 1);
 411         gic_its_write_8(sc, GITS_CBASER, reg);
 412
 413         /* Read back to check for fixed value fields */
 414         tmp = gic_its_read_8(sc, GITS_CBASER);
 415
 416         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 417             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 418                 /* Check if the hardware reported non-shareable */
 419                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 420                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 421                         /* If so remove the cache attribute */
 422                         reg &= ~GITS_CBASER_CACHE_MASK;
 423                         reg &= ~GITS_CBASER_SHARE_MASK;
 424                         /* Set to Non-cacheable, Non-shareable */
 425                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 426                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 427
 428                         gic_its_write_8(sc, GITS_CBASER, reg);
 429                 }
 430
 431                 /* The command queue has to be flushed after each command */
 432                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 433         }
 434
 435         /* Get the next command from the start of the buffer */
 436         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 437 }
 438
 439 static int
 440 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
 441 {
 442         uint64_t reg, tmp;
 443         int page_size;
 444
 445         page_size = PAGE_SIZE_64K;
 446         reg = gic_its_read_8(sc, GITS_BASER(table));
 447
 448         while (1) {
 449                 reg &= GITS_BASER_PSZ_MASK;
 450                 switch (page_size) {
 451                 case PAGE_SIZE_4K:      /* 4KB */
 452                         reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 453                         break;
 454                 case PAGE_SIZE_16K:     /* 16KB */
 455                         reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 456                         break;
 457                 case PAGE_SIZE_64K:     /* 64KB */
 458                         reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 459                         break;
 460                 }
 461
 462                 /* Write the new page size */
 463                 gic_its_write_8(sc, GITS_BASER(table), reg);
 464
 465                 /* Read back to check */
 466                 tmp = gic_its_read_8(sc, GITS_BASER(table));
 467
 468                 /* The page size is correct */
 469                 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
 470                         return (page_size);
 471
 472                 switch (page_size) {
 473                 default:
 474                         return (-1);
 475                 case PAGE_SIZE_16K:
 476                         page_size = PAGE_SIZE_4K;
 477                         break;
 478                 case PAGE_SIZE_64K:
 479                         page_size = PAGE_SIZE_16K;
 480                         break;
 481                 }
 482         }
 483 }
 484
 485 static bool
 486 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table)
 487 {
 488         uint64_t reg;
 489
 490         reg = gic_its_read_8(sc, GITS_BASER(table));
 491
 492         /* Try setting the indirect flag */
 493         reg |= GITS_BASER_INDIRECT;
 494         gic_its_write_8(sc, GITS_BASER(table), reg);
 495
 496         /* Read back to check */
 497         reg = gic_its_read_8(sc, GITS_BASER(table));
 498         return ((reg & GITS_BASER_INDIRECT) != 0);
 499 }
 500
 501
 502 static int
 503 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 504 {
 505         vm_offset_t table;
 506         vm_paddr_t paddr;
 507         uint64_t cache, reg, share, tmp, type;
 508         size_t its_tbl_size, nitspages, npages;
 509         size_t l1_esize, l2_esize, l1_nidents, l2_nidents;
 510         int i, page_size;
 511         int devbits;
 512         bool indirect;
 513
 514         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 515                 /*
 516                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 517                  * are to be 21 bits in length. The entry size of the ITS
 518                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 519                  * is supposed to be 8 bytes in length (for device table).
 520                  * Finally the page size that is to be used by ITS to access
 521                  * this table will be set to 64KB.
 522                  *
 523                  * This gives 0x200000 entries of size 0x8 bytes covered by
 524                  * 256 pages each of which 64KB in size. The number of pages
 525                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 526                  * that case this value would be 0xFF but on ThunderX the
 527                  * maximum value that HW accepts is 0xFD.
 528                  *
 529                  * Set an arbitrary number of device ID bits to 20 in order
 530                  * to limit the number of entries in ITS device table to
 531                  * 0x100000 and the table size to 8MB.
 532                  */
 533                 devbits = 20;
 534                 cache = 0;
 535         } else {
 536                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 537                 cache = GITS_BASER_CACHE_WAWB;
 538         }
 539         sc->sc_devbits = devbits;
 540         share = GITS_BASER_SHARE_IS;
 541
 542         for (i = 0; i < GITS_BASER_NUM; i++) {
 543                 reg = gic_its_read_8(sc, GITS_BASER(i));
 544                 /* The type of table */
 545                 type = GITS_BASER_TYPE(reg);
 546                 if (type == GITS_BASER_TYPE_UNIMPL)
 547                         continue;
 548
 549                 /* The table entry size */
 550                 l1_esize = GITS_BASER_ESIZE(reg);
 551
 552                 /* Find the tables page size */
 553                 page_size = gicv3_its_table_page_size(sc, i);
 554                 if (page_size == -1) {
 555                         device_printf(dev, "No valid page size for table %d\n",
 556                             i);
 557                         return (EINVAL);
 558                 }
 559
 560                 indirect = false;
 561                 l2_nidents = 0;
 562                 l2_esize = 0;
 563                 switch(type) {
 564                 case GITS_BASER_TYPE_DEV:
 565                         if (sc->sc_dev_table_idx != -1)
 566                                 device_printf(dev,
 567                                     "Warning: Multiple device tables found\n");
 568
 569                         sc->sc_dev_table_idx = i;
 570                         l1_nidents = (1 << devbits);
 571                         if ((l1_esize * l1_nidents) > (page_size * 2)) {
 572                                 indirect =
 573                                     gicv3_its_table_supports_indirect(sc, i);
 574                                 if (indirect) {
 575                                         /*
 576                                          * Each l1 entry is 8 bytes and points
 577                                          * to an l2 table of size page_size.
 578                                          * Calculate how many entries this is
 579                                          * and use this to find how many
 580                                          * 8 byte l1 idents we need.
 581                                          */
 582                                         l2_esize = l1_esize;
 583                                         l2_nidents = page_size / l2_esize;
 584                                         l1_nidents = l1_nidents / l2_nidents;
 585                                         l1_esize = GITS_INDIRECT_L1_ESIZE;
 586                                 }
 587                         }
 588                         its_tbl_size = l1_esize * l1_nidents;
 589                         its_tbl_size = roundup2(its_tbl_size, page_size);
 590                         break;
 591                 case GITS_BASER_TYPE_VP:
 592                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 593                 case GITS_BASER_TYPE_IC:
 594                         its_tbl_size = page_size;
 595                         break;
 596                 default:
 597                         if (bootverbose)
 598                                 device_printf(dev, "Unhandled table type %lx\n",
 599                                     type);
 600                         continue;
 601                 }
 602                 npages = howmany(its_tbl_size, PAGE_SIZE);
 603
 604                 /* Allocate the table */
 605                 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
 606                     M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
 607                     (1ul << 48) - 1, PAGE_SIZE_64K, 0);
 608
 609                 sc->sc_its_ptab[i].ptab_vaddr = table;
 610                 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size;
 611                 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents;
 612                 sc->sc_its_ptab[i].ptab_l2_size = page_size;
 613                 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents;
 614
 615                 sc->sc_its_ptab[i].ptab_indirect = indirect;
 616                 sc->sc_its_ptab[i].ptab_page_size = page_size;
 617
 618                 paddr = vtophys(table);
 619
 620                 while (1) {
 621                         nitspages = howmany(its_tbl_size, page_size);
 622
 623                         /* Clear the fields we will be setting */
 624                         reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
 625                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 626                             GITS_BASER_PA_MASK |
 627                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 628                             GITS_BASER_SIZE_MASK);
 629                         /* Set the new values */
 630                         reg |= GITS_BASER_VALID |
 631                             (indirect ? GITS_BASER_INDIRECT : 0) |
 632                             (cache << GITS_BASER_CACHE_SHIFT) |
 633                             (type << GITS_BASER_TYPE_SHIFT) |
 634                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 635                             (nitspages - 1);
 636
 637                         switch (page_size) {
 638                         case PAGE_SIZE_4K:      /* 4KB */
 639                                 reg |=
 640                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 641                                 break;
 642                         case PAGE_SIZE_16K:     /* 16KB */
 643                                 reg |=
 644                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 645                                 break;
 646                         case PAGE_SIZE_64K:     /* 64KB */
 647                                 reg |=
 648                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 649                                 break;
 650                         }
 651
 652                         gic_its_write_8(sc, GITS_BASER(i), reg);
 653
 654                         /* Read back to check */
 655                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 656
 657                         /* Do the shareability masks line up? */
 658                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 659                             (reg & GITS_BASER_SHARE_MASK)) {
 660                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 661                                     GITS_BASER_SHARE_SHIFT;
 662                                 continue;
 663                         }
 664
 665                         if (tmp != reg) {
 666                                 device_printf(dev, "GITS_BASER%d: "
 667                                     "unable to be updated: %lx != %lx\n",
 668                                     i, reg, tmp);
 669                                 return (ENXIO);
 670                         }
 671
 672                         sc->sc_its_ptab[i].ptab_share = share;
 673                         /* We should have made all needed changes */
 674                         break;
 675                 }
 676         }
 677
 678         return (0);
 679 }
 680
 681 static void
 682 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 683 {
 684         void *conf_table;
 685
 686         conf_table = atomic_load_ptr(&conf_base);
 687         if (conf_table == NULL) {
 688                 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
 689                     M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
 690                     LPI_CONFTAB_ALIGN, 0);
 691
 692                 if (atomic_cmpset_ptr((uintptr_t *)&conf_base,
 693                     (uintptr_t)NULL, (uintptr_t)conf_table) == 0) {
 694                         contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS);
 695                         conf_table = atomic_load_ptr(&conf_base);
 696                 }
 697         }
 698         sc->sc_conf_base = conf_table;
 699
 700         /* Set the default configuration */
 701         memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 702             LPI_CONFTAB_SIZE);
 703
 704         /* Flush the table to memory */
 705         cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
 706 }
 707
 708 static void
 709 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 710 {
 711         int i;
 712
 713         for (i = 0; i <= mp_maxid; i++) {
 714                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 715                         continue;
 716
 717                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 718                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 719                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 720
 721                 /* Flush so the ITS can see the memory */
 722                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
 723                     LPI_PENDTAB_SIZE);
 724         }
 725 }
 726
 727 static void
 728 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
 729 {
 730         device_t gicv3;
 731         uint64_t xbaser, tmp;
 732         uint32_t ctlr;
 733         u_int cpuid;
 734
 735         gicv3 = device_get_parent(dev);
 736         cpuid = PCPU_GET(cpuid);
 737
 738         /* Disable LPIs */
 739         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 740         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 741         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 742
 743         /* Make sure changes are observable my the GIC */
 744         dsb(sy);
 745
 746         /*
 747          * Set the redistributor base
 748          */
 749         xbaser = vtophys(sc->sc_conf_base) |
 750             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 751             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 752             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 753         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 754
 755         /* Check the cache attributes we set */
 756         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 757
 758         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 759             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 760                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 761                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 762                         /* We need to mark as non-cacheable */
 763                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 764                             GICR_PROPBASER_CACHE_MASK);
 765                         /* Non-cacheable */
 766                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 767                             GICR_PROPBASER_CACHE_SHIFT;
 768                         /* Non-shareable */
 769                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 770                             GICR_PROPBASER_SHARE_SHIFT;
 771                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 772                 }
 773                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 774         }
 775
 776         /*
 777          * Set the LPI pending table base
 778          */
 779         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 780             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 781             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 782
 783         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 784
 785         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 786
 787         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 788             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 789                 /* Clear the cahce and shareability bits */
 790                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 791                     GICR_PENDBASER_SHARE_MASK);
 792                 /* Mark as non-shareable */
 793                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 794                 /* And non-cacheable */
 795                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 796                     GICR_PENDBASER_CACHE_SHIFT;
 797         }
 798
 799         /* Enable LPIs */
 800         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 801         ctlr |= GICR_CTLR_LPI_ENABLE;
 802         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 803
 804         /* Make sure the GIC has seen everything */
 805         dsb(sy);
 806 }
 807
 808 static int
 809 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 810 {
 811         device_t gicv3;
 812         vm_paddr_t target;
 813         u_int cpuid;
 814         struct redist_pcpu *rpcpu;
 815
 816         gicv3 = device_get_parent(dev);
 817         cpuid = PCPU_GET(cpuid);
 818         if (!CPU_ISSET(cpuid, &sc->sc_cpus))
 819                 return (0);
 820
 821         /* Check if the ITS is enabled on this CPU */
 822         if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
 823                 return (ENXIO);
 824
 825         rpcpu = gicv3_get_redist(dev);
 826
 827         /* Do per-cpu LPI init once */
 828         if (!rpcpu->lpi_enabled) {
 829                 its_init_cpu_lpi(dev, sc);
 830                 rpcpu->lpi_enabled = true;
 831         }
 832
 833         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 834                 /* This ITS wants the redistributor physical address */
 835                 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
 836                     rpcpu->offset);
 837         } else {
 838                 /* This ITS wants the unique processor number */
 839                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
 840                     CMD_TARGET_SHIFT;
 841         }
 842
 843         sc->sc_its_cols[cpuid]->col_target = target;
 844         sc->sc_its_cols[cpuid]->col_id = cpuid;
 845
 846         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 847         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 848
 849         return (0);
 850 }
 851
 852 static int
 853 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
 854 {
 855         struct gicv3_its_softc *sc;
 856         int rv;
 857
 858         sc = arg1;
 859
 860         rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
 861         if (rv != 0 || req->newptr == NULL)
 862                 return (rv);
 863         if (sc->trace_enable)
 864                 gic_its_write_8(sc, GITS_TRKCTLR, 3);
 865         else
 866                 gic_its_write_8(sc, GITS_TRKCTLR, 0);
 867
 868         return (0);
 869 }
 870
 871 static int
 872 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
 873 {
 874         struct gicv3_its_softc *sc;
 875         struct sbuf *sb;
 876         int err;
 877
 878         sc = arg1;
 879         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 880         if (sb == NULL) {
 881                 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
 882                 return (ENOMEM);
 883         }
 884         sbuf_cat(sb, "\n");
 885         sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
 886             gic_its_read_4(sc, GITS_TRKCTLR));
 887         sbuf_printf(sb, "GITS_TRKR:    0x%08X\n",
 888             gic_its_read_4(sc, GITS_TRKR));
 889         sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
 890             gic_its_read_4(sc, GITS_TRKDIDR));
 891         sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
 892             gic_its_read_4(sc, GITS_TRKPIDR));
 893         sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
 894             gic_its_read_4(sc, GITS_TRKVIDR));
 895         sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
 896            gic_its_read_4(sc, GITS_TRKTGTR));
 897
 898         err = sbuf_finish(sb);
 899         if (err)
 900                 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
 901         sbuf_delete(sb);
 902         return(err);
 903 }
 904
 905 static int
 906 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
 907 {
 908         struct sysctl_oid *oid, *child;
 909         struct sysctl_ctx_list *ctx_list;
 910
 911         ctx_list = device_get_sysctl_ctx(sc->dev);
 912         child = device_get_sysctl_tree(sc->dev);
 913         oid = SYSCTL_ADD_NODE(ctx_list,
 914             SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
 915             CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
 916         if (oid == NULL)
 917                 return (ENXIO);
 918
 919         /* Add registers */
 920         SYSCTL_ADD_PROC(ctx_list,
 921             SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
 922             CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 923             gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
 924         SYSCTL_ADD_PROC(ctx_list,
 925             SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
 926             CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 927             gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
 928
 929         return (0);
 930 }
 931
 932 static int
 933 gicv3_its_attach(device_t dev)
 934 {
 935         struct gicv3_its_softc *sc;
 936         int domain, err, i, rid;
 937         uint64_t phys;
 938         uint32_t ctlr, iidr;
 939
 940         sc = device_get_softc(dev);
 941
 942         sc->sc_dev_table_idx = -1;
 943         sc->sc_irq_length = gicv3_get_nirqs(dev);
 944         sc->sc_irq_base = GIC_FIRST_LPI;
 945         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
 946
 947         rid = 0;
 948         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 949             RF_ACTIVE);
 950         if (sc->sc_its_res == NULL) {
 951                 device_printf(dev, "Could not allocate memory\n");
 952                 return (ENXIO);
 953         }
 954
 955         phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
 956             GITS_TRANSLATER, PAGE_SIZE);
 957         sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
 958         vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
 959
 960         CPU_COPY(&all_cpus, &sc->sc_cpus);
 961         iidr = gic_its_read_4(sc, GITS_IIDR);
 962         for (i = 0; i < nitems(its_quirks); i++) {
 963                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 964                         if (bootverbose) {
 965                                 device_printf(dev, "Applying %s\n",
 966                                     its_quirks[i].desc);
 967                         }
 968                         its_quirks[i].func(dev);
 969                         break;
 970                 }
 971         }
 972
 973         if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
 974                 sc->sc_ds = DOMAINSET_PREF(domain);
 975         } else {
 976                 sc->sc_ds = DOMAINSET_RR();
 977         }
 978
 979         /*
 980          * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
 981          * coming in via, for instance, a kexec/kboot style setup where a
 982          * previous kernel has configured then relinquished control.  Clear it
 983          * so that we can reconfigure GITS_BASER*.
 984          */
 985         ctlr = gic_its_read_4(sc, GITS_CTLR);
 986         if ((ctlr & GITS_CTLR_EN) != 0) {
 987                 ctlr &= ~GITS_CTLR_EN;
 988                 gic_its_write_4(sc, GITS_CTLR, ctlr);
 989         }
 990
 991         /* Allocate the private tables */
 992         err = gicv3_its_table_init(dev, sc);
 993         if (err != 0)
 994                 return (err);
 995
 996         /* Protects access to the device list */
 997         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 998
 999         /* Protects access to the ITS command circular buffer. */
1000         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
1001
1002         /* Allocate the command circular buffer */
1003         gicv3_its_cmdq_init(sc);
1004
1005         /* Allocate the per-CPU collections */
1006         for (int cpu = 0; cpu <= mp_maxid; cpu++)
1007                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
1008                         sc->sc_its_cols[cpu] = malloc_domainset(
1009                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
1010                             DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
1011                             M_WAITOK | M_ZERO);
1012                 else
1013                         sc->sc_its_cols[cpu] = NULL;
1014
1015         /* Enable the ITS */
1016         gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
1017
1018         /* Create the LPI configuration table */
1019         gicv3_its_conftable_init(sc);
1020
1021         /* And the pending tebles */
1022         gicv3_its_pendtables_init(sc);
1023
1024         /* Enable LPIs on this CPU */
1025         its_init_cpu(dev, sc);
1026
1027         TAILQ_INIT(&sc->sc_its_dev_list);
1028         TAILQ_INIT(&sc->sc_free_irqs);
1029
1030         /*
1031          * Create the vmem object to allocate INTRNG IRQs from. We try to
1032          * use all IRQs not already used by the GICv3.
1033          * XXX: This assumes there are no other interrupt controllers in the
1034          * system.
1035          */
1036         sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
1037             gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
1038
1039         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
1040             M_GICV3_ITS, M_WAITOK | M_ZERO);
1041
1042         /* For GIC-500 install tracking sysctls. */
1043         if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
1044             GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
1045                 gicv3_its_init_sysctl(sc);
1046
1047         return (0);
1048 }
1049
1050 static int
1051 gicv3_its_detach(device_t dev)
1052 {
1053
1054         return (ENXIO);
1055 }
1056
1057 static void
1058 its_quirk_cavium_22375(device_t dev)
1059 {
1060         struct gicv3_its_softc *sc;
1061         int domain;
1062
1063         sc = device_get_softc(dev);
1064         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
1065
1066         /*
1067          * We need to limit which CPUs we send these interrupts to on
1068          * the original dual socket ThunderX as it is unable to
1069          * forward them between the two sockets.
1070          */
1071         if (bus_get_domain(dev, &domain) == 0) {
1072                 if (domain < MAXMEMDOM) {
1073                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1074                 } else {
1075                         CPU_ZERO(&sc->sc_cpus);
1076                 }
1077         }
1078 }
1079
1080 static void
1081 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1082 {
1083         struct gicv3_its_softc *sc;
1084         struct gicv3_its_irqsrc *girq;
1085         uint8_t *conf;
1086
1087         sc = device_get_softc(dev);
1088         girq = (struct gicv3_its_irqsrc *)isrc;
1089         conf = sc->sc_conf_base;
1090
1091         conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1092
1093         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1094                 /* Clean D-cache under command. */
1095                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1096         } else {
1097                 /* DSB inner shareable, store */
1098                 dsb(ishst);
1099         }
1100
1101         its_cmd_inv(dev, girq->gi_its_dev, girq);
1102 }
1103
1104 static void
1105 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1106 {
1107         struct gicv3_its_softc *sc;
1108         struct gicv3_its_irqsrc *girq;
1109         uint8_t *conf;
1110
1111         sc = device_get_softc(dev);
1112         girq = (struct gicv3_its_irqsrc *)isrc;
1113         conf = sc->sc_conf_base;
1114
1115         conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1116
1117         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1118                 /* Clean D-cache under command. */
1119                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1120         } else {
1121                 /* DSB inner shareable, store */
1122                 dsb(ishst);
1123         }
1124
1125         its_cmd_inv(dev, girq->gi_its_dev, girq);
1126 }
1127
1128 static int
1129 gicv3_its_intr(void *arg, uintptr_t irq)
1130 {
1131         struct gicv3_its_softc *sc = arg;
1132         struct gicv3_its_irqsrc *girq;
1133         struct trapframe *tf;
1134
1135         irq -= sc->sc_irq_base;
1136         girq = sc->sc_irqs[irq];
1137         if (girq == NULL)
1138                 panic("gicv3_its_intr: Invalid interrupt %ld",
1139                     irq + sc->sc_irq_base);
1140
1141         tf = curthread->td_intr_frame;
1142         intr_isrc_dispatch(&girq->gi_isrc, tf);
1143         return (FILTER_HANDLED);
1144 }
1145
1146 static void
1147 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1148 {
1149         struct gicv3_its_irqsrc *girq;
1150
1151         girq = (struct gicv3_its_irqsrc *)isrc;
1152         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1153 }
1154
1155 static void
1156 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1157 {
1158
1159 }
1160
1161 static void
1162 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1163 {
1164         struct gicv3_its_irqsrc *girq;
1165
1166         girq = (struct gicv3_its_irqsrc *)isrc;
1167         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1168 }
1169
1170 static int
1171 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1172 {
1173         struct gicv3_its_softc *sc;
1174
1175         sc = device_get_softc(dev);
1176         if (CPU_EMPTY(&isrc->isrc_cpu)) {
1177                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1178                     &sc->sc_cpus);
1179                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1180         }
1181
1182         return (0);
1183 }
1184
1185 static int
1186 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1187 {
1188         struct gicv3_its_irqsrc *girq;
1189
1190         gicv3_its_select_cpu(dev, isrc);
1191
1192         girq = (struct gicv3_its_irqsrc *)isrc;
1193         its_cmd_movi(dev, girq);
1194         return (0);
1195 }
1196
1197 static int
1198 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1199     struct intr_irqsrc **isrcp)
1200 {
1201
1202         /*
1203          * This should never happen, we only call this function to map
1204          * interrupts found before the controller driver is ready.
1205          */
1206         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1207 }
1208
1209 static int
1210 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1211     struct resource *res, struct intr_map_data *data)
1212 {
1213
1214         /* Bind the interrupt to a CPU */
1215         gicv3_its_bind_intr(dev, isrc);
1216
1217         return (0);
1218 }
1219
1220 #ifdef SMP
1221 static void
1222 gicv3_its_init_secondary(device_t dev)
1223 {
1224         struct gicv3_its_softc *sc;
1225
1226         sc = device_get_softc(dev);
1227
1228         /*
1229          * This is fatal as otherwise we may bind interrupts to this CPU.
1230          * We need a way to tell the interrupt framework to only bind to a
1231          * subset of given CPUs when it performs the shuffle.
1232          */
1233         if (its_init_cpu(dev, sc) != 0)
1234                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1235                     PCPU_GET(cpuid));
1236 }
1237 #endif
1238
1239 static uint32_t
1240 its_get_devid(device_t pci_dev)
1241 {
1242         uintptr_t id;
1243
1244         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1245                 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1246                     device_get_nameunit(pci_dev));
1247
1248         return (id);
1249 }
1250
1251 static struct its_dev *
1252 its_device_find(device_t dev, device_t child)
1253 {
1254         struct gicv3_its_softc *sc;
1255         struct its_dev *its_dev = NULL;
1256
1257         sc = device_get_softc(dev);
1258
1259         mtx_lock_spin(&sc->sc_its_dev_lock);
1260         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1261                 if (its_dev->pci_dev == child)
1262                         break;
1263         }
1264         mtx_unlock_spin(&sc->sc_its_dev_lock);
1265
1266         return (its_dev);
1267 }
1268
1269 static bool
1270 its_device_alloc(struct gicv3_its_softc *sc, int devid)
1271 {
1272         struct its_ptable *ptable;
1273         vm_offset_t l2_table;
1274         uint64_t *table;
1275         uint32_t index;
1276         bool shareable;
1277
1278         /* No device table */
1279         if (sc->sc_dev_table_idx < 0) {
1280                 if (devid >= (1 << sc->sc_devbits)) {
1281                         if (bootverbose) {
1282                                 device_printf(sc->dev,
1283                                     "%s: Device out of range for hardware "
1284                                     "(%x >= %x)\n", __func__, devid,
1285                                     1 << sc->sc_devbits);
1286                         }
1287                         return (false);
1288                 }
1289                 return (true);
1290         }
1291
1292         ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx];
1293         /* Check the devid is within the table limit */
1294         if (!ptable->ptab_indirect) {
1295                 if (devid >= ptable->ptab_l1_nidents) {
1296                         if (bootverbose) {
1297                                 device_printf(sc->dev,
1298                                     "%s: Device out of range for table "
1299                                     "(%x >= %x)\n", __func__, devid,
1300                                     ptable->ptab_l1_nidents);
1301                         }
1302                         return (false);
1303                 }
1304
1305                 return (true);
1306         }
1307
1308         /* Check the devid is within the allocated range */
1309         index = devid / ptable->ptab_l2_nidents;
1310         if (index >= ptable->ptab_l1_nidents) {
1311                 if (bootverbose) {
1312                         device_printf(sc->dev,
1313                             "%s: Index out of range for table (%x >= %x)\n",
1314                             __func__, index, ptable->ptab_l1_nidents);
1315                 }
1316                 return (false);
1317         }
1318
1319         table = (uint64_t *)ptable->ptab_vaddr;
1320         /* We have an second level table */
1321         if ((table[index] & GITS_BASER_VALID) != 0)
1322                 return (true);
1323
1324         shareable = true;
1325         if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS)
1326                 shareable = false;
1327
1328         l2_table = (vm_offset_t)contigmalloc_domainset(ptable->ptab_l2_size,
1329             M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
1330             ptable->ptab_page_size, 0);
1331
1332         if (!shareable)
1333                 cpu_dcache_wb_range((vm_offset_t)l2_table,
1334                     ptable->ptab_l2_size);
1335
1336         table[index] = vtophys(l2_table) | GITS_BASER_VALID;
1337         if (!shareable)
1338                 cpu_dcache_wb_range((vm_offset_t)&table[index],
1339                     sizeof(table[index]));
1340
1341         dsb(sy);
1342         return (true);
1343 }
1344
1345 static struct its_dev *
1346 its_device_get(device_t dev, device_t child, u_int nvecs)
1347 {
1348         struct gicv3_its_softc *sc;
1349         struct its_dev *its_dev;
1350         vmem_addr_t irq_base;
1351         size_t esize;
1352
1353         sc = device_get_softc(dev);
1354
1355         its_dev = its_device_find(dev, child);
1356         if (its_dev != NULL)
1357                 return (its_dev);
1358
1359         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1360         if (its_dev == NULL)
1361                 return (NULL);
1362
1363         its_dev->pci_dev = child;
1364         its_dev->devid = its_get_devid(child);
1365
1366         its_dev->lpis.lpi_busy = 0;
1367         its_dev->lpis.lpi_num = nvecs;
1368         its_dev->lpis.lpi_free = nvecs;
1369
1370         if (!its_device_alloc(sc, its_dev->devid)) {
1371                 free(its_dev, M_GICV3_ITS);
1372                 return (NULL);
1373         }
1374
1375         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1376             &irq_base) != 0) {
1377                 free(its_dev, M_GICV3_ITS);
1378                 return (NULL);
1379         }
1380         its_dev->lpis.lpi_base = irq_base;
1381
1382         /* Get ITT entry size */
1383         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1384
1385         /*
1386          * Allocate ITT for this device.
1387          * PA has to be 256 B aligned. At least two entries for device.
1388          */
1389         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1390         its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
1391             M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1392             LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1393         if (its_dev->itt == 0) {
1394                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1395                 free(its_dev, M_GICV3_ITS);
1396                 return (NULL);
1397         }
1398
1399         /* Make sure device sees zeroed ITT. */
1400         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1401                 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1402
1403         mtx_lock_spin(&sc->sc_its_dev_lock);
1404         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1405         mtx_unlock_spin(&sc->sc_its_dev_lock);
1406
1407         /* Map device to its ITT */
1408         its_cmd_mapd(dev, its_dev, 1);
1409
1410         return (its_dev);
1411 }
1412
1413 static void
1414 its_device_release(device_t dev, struct its_dev *its_dev)
1415 {
1416         struct gicv3_its_softc *sc;
1417
1418         KASSERT(its_dev->lpis.lpi_busy == 0,
1419             ("its_device_release: Trying to release an inuse ITS device"));
1420
1421         /* Unmap device in ITS */
1422         its_cmd_mapd(dev, its_dev, 0);
1423
1424         sc = device_get_softc(dev);
1425
1426         /* Remove the device from the list of devices */
1427         mtx_lock_spin(&sc->sc_its_dev_lock);
1428         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1429         mtx_unlock_spin(&sc->sc_its_dev_lock);
1430
1431         /* Free ITT */
1432         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1433         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1434
1435         /* Free the IRQ allocation */
1436         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1437             its_dev->lpis.lpi_num);
1438
1439         free(its_dev, M_GICV3_ITS);
1440 }
1441
1442 static struct gicv3_its_irqsrc *
1443 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1444 {
1445         struct gicv3_its_irqsrc *girq = NULL;
1446
1447         KASSERT(sc->sc_irqs[irq] == NULL,
1448             ("%s: Interrupt %u already allocated", __func__, irq));
1449         mtx_lock_spin(&sc->sc_its_dev_lock);
1450         if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1451                 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1452                 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1453         }
1454         mtx_unlock_spin(&sc->sc_its_dev_lock);
1455         if (girq == NULL) {
1456                 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1457                     M_NOWAIT | M_ZERO);
1458                 if (girq == NULL)
1459                         return (NULL);
1460                 girq->gi_id = -1;
1461                 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1462                     "%s,%u", device_get_nameunit(dev), irq) != 0) {
1463                         free(girq, M_GICV3_ITS);
1464                         return (NULL);
1465                 }
1466         }
1467         girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1468         sc->sc_irqs[irq] = girq;
1469
1470         return (girq);
1471 }
1472
1473 static void
1474 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1475     struct gicv3_its_irqsrc *girq)
1476 {
1477         u_int irq;
1478
1479         mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1480
1481         irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1482         sc->sc_irqs[irq] = NULL;
1483
1484         girq->gi_id = -1;
1485         girq->gi_its_dev = NULL;
1486         TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1487 }
1488
1489 static int
1490 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1491     device_t *pic, struct intr_irqsrc **srcs)
1492 {
1493         struct gicv3_its_softc *sc;
1494         struct gicv3_its_irqsrc *girq;
1495         struct its_dev *its_dev;
1496         u_int irq;
1497         int i;
1498
1499         its_dev = its_device_get(dev, child, count);
1500         if (its_dev == NULL)
1501                 return (ENXIO);
1502
1503         KASSERT(its_dev->lpis.lpi_free >= count,
1504             ("gicv3_its_alloc_msi: No free LPIs"));
1505         sc = device_get_softc(dev);
1506         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1507             its_dev->lpis.lpi_free;
1508
1509         /* Allocate the irqsrc for each MSI */
1510         for (i = 0; i < count; i++, irq++) {
1511                 its_dev->lpis.lpi_free--;
1512                 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1513                     sc, irq);
1514                 if (srcs[i] == NULL)
1515                         break;
1516         }
1517
1518         /* The allocation failed, release them */
1519         if (i != count) {
1520                 mtx_lock_spin(&sc->sc_its_dev_lock);
1521                 for (i = 0; i < count; i++) {
1522                         girq = (struct gicv3_its_irqsrc *)srcs[i];
1523                         if (girq == NULL)
1524                                 break;
1525                         gicv3_its_release_irqsrc(sc, girq);
1526                         srcs[i] = NULL;
1527                 }
1528                 mtx_unlock_spin(&sc->sc_its_dev_lock);
1529                 return (ENXIO);
1530         }
1531
1532         /* Finish the allocation now we have all MSI irqsrcs */
1533         for (i = 0; i < count; i++) {
1534                 girq = (struct gicv3_its_irqsrc *)srcs[i];
1535                 girq->gi_id = i;
1536                 girq->gi_its_dev = its_dev;
1537
1538                 /* Map the message to the given IRQ */
1539                 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1540                 its_cmd_mapti(dev, girq);
1541         }
1542         its_dev->lpis.lpi_busy += count;
1543         *pic = dev;
1544
1545         return (0);
1546 }
1547
1548 static int
1549 gicv3_its_release_msi(device_t dev, device_t child, int count,
1550     struct intr_irqsrc **isrc)
1551 {
1552         struct gicv3_its_softc *sc;
1553         struct gicv3_its_irqsrc *girq;
1554         struct its_dev *its_dev;
1555         int i;
1556
1557         its_dev = its_device_find(dev, child);
1558
1559         KASSERT(its_dev != NULL,
1560             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1561              "no ITS device"));
1562         KASSERT(its_dev->lpis.lpi_busy >= count,
1563             ("gicv3_its_release_msi: Releasing more interrupts than "
1564              "were allocated: releasing %d, allocated %d", count,
1565              its_dev->lpis.lpi_busy));
1566
1567         sc = device_get_softc(dev);
1568         mtx_lock_spin(&sc->sc_its_dev_lock);
1569         for (i = 0; i < count; i++) {
1570                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1571                 gicv3_its_release_irqsrc(sc, girq);
1572         }
1573         mtx_unlock_spin(&sc->sc_its_dev_lock);
1574         its_dev->lpis.lpi_busy -= count;
1575
1576         if (its_dev->lpis.lpi_busy == 0)
1577                 its_device_release(dev, its_dev);
1578
1579         return (0);
1580 }
1581
1582 static int
1583 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1584     struct intr_irqsrc **isrcp)
1585 {
1586         struct gicv3_its_softc *sc;
1587         struct gicv3_its_irqsrc *girq;
1588         struct its_dev *its_dev;
1589         u_int nvecs, irq;
1590
1591         nvecs = pci_msix_count(child);
1592         its_dev = its_device_get(dev, child, nvecs);
1593         if (its_dev == NULL)
1594                 return (ENXIO);
1595
1596         KASSERT(its_dev->lpis.lpi_free > 0,
1597             ("gicv3_its_alloc_msix: No free LPIs"));
1598         sc = device_get_softc(dev);
1599         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1600             its_dev->lpis.lpi_free;
1601
1602         girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1603         if (girq == NULL)
1604                 return (ENXIO);
1605         girq->gi_id = its_dev->lpis.lpi_busy;
1606         girq->gi_its_dev = its_dev;
1607
1608         its_dev->lpis.lpi_free--;
1609         its_dev->lpis.lpi_busy++;
1610
1611         /* Map the message to the given IRQ */
1612         gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1613         its_cmd_mapti(dev, girq);
1614
1615         *pic = dev;
1616         *isrcp = (struct intr_irqsrc *)girq;
1617
1618         return (0);
1619 }
1620
1621 static int
1622 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1623 {
1624         struct gicv3_its_softc *sc;
1625         struct gicv3_its_irqsrc *girq;
1626         struct its_dev *its_dev;
1627
1628         its_dev = its_device_find(dev, child);
1629
1630         KASSERT(its_dev != NULL,
1631             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1632              "no ITS device"));
1633         KASSERT(its_dev->lpis.lpi_busy > 0,
1634             ("gicv3_its_release_msix: Releasing more interrupts than "
1635              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1636
1637         sc = device_get_softc(dev);
1638         girq = (struct gicv3_its_irqsrc *)isrc;
1639         mtx_lock_spin(&sc->sc_its_dev_lock);
1640         gicv3_its_release_irqsrc(sc, girq);
1641         mtx_unlock_spin(&sc->sc_its_dev_lock);
1642         its_dev->lpis.lpi_busy--;
1643
1644         if (its_dev->lpis.lpi_busy == 0)
1645                 its_device_release(dev, its_dev);
1646
1647         return (0);
1648 }
1649
1650 static int
1651 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1652     uint64_t *addr, uint32_t *data)
1653 {
1654         struct gicv3_its_softc *sc;
1655         struct gicv3_its_irqsrc *girq;
1656
1657         sc = device_get_softc(dev);
1658         girq = (struct gicv3_its_irqsrc *)isrc;
1659
1660         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1661         *data = girq->gi_id;
1662
1663         return (0);
1664 }
1665
1666 #ifdef IOMMU
1667 static int
1668 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1669 {
1670         struct gicv3_its_softc *sc;
1671         struct iommu_ctx *ctx;
1672         int error;
1673
1674         sc = device_get_softc(dev);
1675         ctx = iommu_get_dev_ctx(child);
1676         if (ctx == NULL)
1677                 return (ENXIO);
1678         /* Map the page containing the GITS_TRANSLATER register. */
1679         error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1680             IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1681         *domain = iommu_get_ctx_domain(ctx);
1682
1683         return (error);
1684 }
1685
1686 static void
1687 gicv3_iommu_deinit(device_t dev, device_t child)
1688 {
1689         struct iommu_ctx *ctx;
1690
1691         ctx = iommu_get_dev_ctx(child);
1692         if (ctx == NULL)
1693                 return;
1694
1695         iommu_unmap_msi(ctx);
1696 }
1697 #endif
1698
1699 /*
1700  * Commands handling.
1701  */
1702
1703 static __inline void
1704 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1705 {
1706         /* Command field: DW0 [7:0] */
1707         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1708         cmd->cmd_dword[0] |= htole64(cmd_type);
1709 }
1710
1711 static __inline void
1712 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1713 {
1714         /* Device ID field: DW0 [63:32] */
1715         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1716         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1717 }
1718
1719 static __inline void
1720 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1721 {
1722         /* Size field: DW1 [4:0] */
1723         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1724         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1725 }
1726
1727 static __inline void
1728 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1729 {
1730         /* ID field: DW1 [31:0] */
1731         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1732         cmd->cmd_dword[1] |= htole64(id);
1733 }
1734
1735 static __inline void
1736 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1737 {
1738         /* Physical ID field: DW1 [63:32] */
1739         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1740         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1741 }
1742
1743 static __inline void
1744 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1745 {
1746         /* Collection field: DW2 [16:0] */
1747         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1748         cmd->cmd_dword[2] |= htole64(col_id);
1749 }
1750
1751 static __inline void
1752 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1753 {
1754         /* Target Address field: DW2 [47:16] */
1755         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1756         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1757 }
1758
1759 static __inline void
1760 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1761 {
1762         /* ITT Address field: DW2 [47:8] */
1763         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1764         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1765 }
1766
1767 static __inline void
1768 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1769 {
1770         /* Valid field: DW2 [63] */
1771         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1772         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1773 }
1774
1775 static inline bool
1776 its_cmd_queue_full(struct gicv3_its_softc *sc)
1777 {
1778         size_t read_idx, next_write_idx;
1779
1780         /* Get the index of the next command */
1781         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1782             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1783         /* And the index of the current command being read */
1784         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1785
1786         /*
1787          * The queue is full when the write offset points
1788          * at the command before the current read offset.
1789          */
1790         return (next_write_idx == read_idx);
1791 }
1792
1793 static inline void
1794 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1795 {
1796
1797         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1798                 /* Clean D-cache under command. */
1799                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1800         } else {
1801                 /* DSB inner shareable, store */
1802                 dsb(ishst);
1803         }
1804
1805 }
1806
1807 static inline uint64_t
1808 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1809 {
1810         uint64_t off;
1811
1812         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1813
1814         return (off);
1815 }
1816
1817 static void
1818 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1819     struct its_cmd *cmd_last)
1820 {
1821         struct gicv3_its_softc *sc;
1822         uint64_t first, last, read;
1823         size_t us_left;
1824
1825         sc = device_get_softc(dev);
1826
1827         /*
1828          * XXX ARM64TODO: This is obviously a significant delay.
1829          * The reason for that is that currently the time frames for
1830          * the command to complete are not known.
1831          */
1832         us_left = 1000000;
1833
1834         first = its_cmd_cwriter_offset(sc, cmd_first);
1835         last = its_cmd_cwriter_offset(sc, cmd_last);
1836
1837         for (;;) {
1838                 read = gic_its_read_8(sc, GITS_CREADR);
1839                 if (first < last) {
1840                         if (read < first || read >= last)
1841                                 break;
1842                 } else if (read < first && read >= last)
1843                         break;
1844
1845                 if (us_left-- == 0) {
1846                         /* This means timeout */
1847                         device_printf(dev,
1848                             "Timeout while waiting for CMD completion.\n");
1849                         return;
1850                 }
1851                 DELAY(1);
1852         }
1853 }
1854
1855 static struct its_cmd *
1856 its_cmd_alloc_locked(device_t dev)
1857 {
1858         struct gicv3_its_softc *sc;
1859         struct its_cmd *cmd;
1860         size_t us_left;
1861
1862         sc = device_get_softc(dev);
1863
1864         /*
1865          * XXX ARM64TODO: This is obviously a significant delay.
1866          * The reason for that is that currently the time frames for
1867          * the command to complete (and therefore free the descriptor)
1868          * are not known.
1869          */
1870         us_left = 1000000;
1871
1872         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1873         while (its_cmd_queue_full(sc)) {
1874                 if (us_left-- == 0) {
1875                         /* Timeout while waiting for free command */
1876                         device_printf(dev,
1877                             "Timeout while waiting for free command\n");
1878                         return (NULL);
1879                 }
1880                 DELAY(1);
1881         }
1882
1883         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1884         sc->sc_its_cmd_next_idx++;
1885         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1886
1887         return (cmd);
1888 }
1889
1890 static uint64_t
1891 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1892 {
1893         uint64_t target;
1894         uint8_t cmd_type;
1895         u_int size;
1896
1897         cmd_type = desc->cmd_type;
1898         target = ITS_TARGET_NONE;
1899
1900         switch (cmd_type) {
1901         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1902                 target = desc->cmd_desc_movi.col->col_target;
1903                 cmd_format_command(cmd, ITS_CMD_MOVI);
1904                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1905                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1906                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1907                 break;
1908         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1909                 target = desc->cmd_desc_sync.col->col_target;
1910                 cmd_format_command(cmd, ITS_CMD_SYNC);
1911                 cmd_format_target(cmd, target);
1912                 break;
1913         case ITS_CMD_MAPD:      /* Assign ITT to device */
1914                 cmd_format_command(cmd, ITS_CMD_MAPD);
1915                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1916                 /*
1917                  * Size describes number of bits to encode interrupt IDs
1918                  * supported by the device minus one.
1919                  * When V (valid) bit is zero, this field should be written
1920                  * as zero.
1921                  */
1922                 if (desc->cmd_desc_mapd.valid != 0) {
1923                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1924                         size = MAX(1, size) - 1;
1925                 } else
1926                         size = 0;
1927
1928                 cmd_format_size(cmd, size);
1929                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1930                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1931                 break;
1932         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1933                 target = desc->cmd_desc_mapc.col->col_target;
1934                 cmd_format_command(cmd, ITS_CMD_MAPC);
1935                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1936                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1937                 cmd_format_target(cmd, target);
1938                 break;
1939         case ITS_CMD_MAPTI:
1940                 target = desc->cmd_desc_mapvi.col->col_target;
1941                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1942                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1943                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1944                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1945                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1946                 break;
1947         case ITS_CMD_MAPI:
1948                 target = desc->cmd_desc_mapi.col->col_target;
1949                 cmd_format_command(cmd, ITS_CMD_MAPI);
1950                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1951                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1952                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1953                 break;
1954         case ITS_CMD_INV:
1955                 target = desc->cmd_desc_inv.col->col_target;
1956                 cmd_format_command(cmd, ITS_CMD_INV);
1957                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1958                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1959                 break;
1960         case ITS_CMD_INVALL:
1961                 cmd_format_command(cmd, ITS_CMD_INVALL);
1962                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1963                 break;
1964         default:
1965                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1966         }
1967
1968         return (target);
1969 }
1970
1971 static int
1972 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1973 {
1974         struct gicv3_its_softc *sc;
1975         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1976         struct its_col col_sync;
1977         struct its_cmd_desc desc_sync;
1978         uint64_t target, cwriter;
1979
1980         sc = device_get_softc(dev);
1981         mtx_lock_spin(&sc->sc_its_cmd_lock);
1982         cmd = its_cmd_alloc_locked(dev);
1983         if (cmd == NULL) {
1984                 device_printf(dev, "could not allocate ITS command\n");
1985                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1986                 return (EBUSY);
1987         }
1988
1989         target = its_cmd_prepare(cmd, desc);
1990         its_cmd_sync(sc, cmd);
1991
1992         if (target != ITS_TARGET_NONE) {
1993                 cmd_sync = its_cmd_alloc_locked(dev);
1994                 if (cmd_sync != NULL) {
1995                         desc_sync.cmd_type = ITS_CMD_SYNC;
1996                         col_sync.col_target = target;
1997                         desc_sync.cmd_desc_sync.col = &col_sync;
1998                         its_cmd_prepare(cmd_sync, &desc_sync);
1999                         its_cmd_sync(sc, cmd_sync);
2000                 }
2001         }
2002
2003         /* Update GITS_CWRITER */
2004         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
2005         gic_its_write_8(sc, GITS_CWRITER, cwriter);
2006         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2007         mtx_unlock_spin(&sc->sc_its_cmd_lock);
2008
2009         its_cmd_wait_completion(dev, cmd, cmd_write);
2010
2011         return (0);
2012 }
2013
2014 /* Handlers to send commands */
2015 static void
2016 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
2017 {
2018         struct gicv3_its_softc *sc;
2019         struct its_cmd_desc desc;
2020         struct its_col *col;
2021
2022         sc = device_get_softc(dev);
2023         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2024
2025         desc.cmd_type = ITS_CMD_MOVI;
2026         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
2027         desc.cmd_desc_movi.col = col;
2028         desc.cmd_desc_movi.id = girq->gi_id;
2029
2030         its_cmd_send(dev, &desc);
2031 }
2032
2033 static void
2034 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
2035 {
2036         struct its_cmd_desc desc;
2037
2038         desc.cmd_type = ITS_CMD_MAPC;
2039         desc.cmd_desc_mapc.col = col;
2040         /*
2041          * Valid bit set - map the collection.
2042          * Valid bit cleared - unmap the collection.
2043          */
2044         desc.cmd_desc_mapc.valid = valid;
2045
2046         its_cmd_send(dev, &desc);
2047 }
2048
2049 static void
2050 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
2051 {
2052         struct gicv3_its_softc *sc;
2053         struct its_cmd_desc desc;
2054         struct its_col *col;
2055         u_int col_id;
2056
2057         sc = device_get_softc(dev);
2058
2059         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
2060         col = sc->sc_its_cols[col_id];
2061
2062         desc.cmd_type = ITS_CMD_MAPTI;
2063         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
2064         desc.cmd_desc_mapvi.col = col;
2065         /* The EventID sent to the device */
2066         desc.cmd_desc_mapvi.id = girq->gi_id;
2067         /* The physical interrupt presented to softeware */
2068         desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
2069
2070         its_cmd_send(dev, &desc);
2071 }
2072
2073 static void
2074 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
2075 {
2076         struct its_cmd_desc desc;
2077
2078         desc.cmd_type = ITS_CMD_MAPD;
2079         desc.cmd_desc_mapd.its_dev = its_dev;
2080         desc.cmd_desc_mapd.valid = valid;
2081
2082         its_cmd_send(dev, &desc);
2083 }
2084
2085 static void
2086 its_cmd_inv(device_t dev, struct its_dev *its_dev,
2087     struct gicv3_its_irqsrc *girq)
2088 {
2089         struct gicv3_its_softc *sc;
2090         struct its_cmd_desc desc;
2091         struct its_col *col;
2092
2093         sc = device_get_softc(dev);
2094         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2095
2096         desc.cmd_type = ITS_CMD_INV;
2097         /* The EventID sent to the device */
2098         desc.cmd_desc_inv.pid = girq->gi_id;
2099         desc.cmd_desc_inv.its_dev = its_dev;
2100         desc.cmd_desc_inv.col = col;
2101
2102         its_cmd_send(dev, &desc);
2103 }
2104
2105 static void
2106 its_cmd_invall(device_t dev, struct its_col *col)
2107 {
2108         struct its_cmd_desc desc;
2109
2110         desc.cmd_type = ITS_CMD_INVALL;
2111         desc.cmd_desc_invall.col = col;
2112
2113         its_cmd_send(dev, &desc);
2114 }
2115
2116 #ifdef FDT
2117 static device_probe_t gicv3_its_fdt_probe;
2118 static device_attach_t gicv3_its_fdt_attach;
2119
2120 static device_method_t gicv3_its_fdt_methods[] = {
2121         /* Device interface */
2122         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
2123         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
2124
2125         /* End */
2126         DEVMETHOD_END
2127 };
2128
2129 #define its_baseclasses its_fdt_baseclasses
2130 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
2131     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2132 #undef its_baseclasses
2133
2134 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
2135     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2136
2137 static int
2138 gicv3_its_fdt_probe(device_t dev)
2139 {
2140
2141         if (!ofw_bus_status_okay(dev))
2142                 return (ENXIO);
2143
2144         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
2145                 return (ENXIO);
2146
2147         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2148         return (BUS_PROBE_DEFAULT);
2149 }
2150
2151 static int
2152 gicv3_its_fdt_attach(device_t dev)
2153 {
2154         struct gicv3_its_softc *sc;
2155         phandle_t xref;
2156         int err;
2157
2158         sc = device_get_softc(dev);
2159         sc->dev = dev;
2160         err = gicv3_its_attach(dev);
2161         if (err != 0)
2162                 return (err);
2163
2164         /* Register this device as a interrupt controller */
2165         xref = OF_xref_from_node(ofw_bus_get_node(dev));
2166         sc->sc_pic = intr_pic_register(dev, xref);
2167         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2168             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2169         if (err != 0) {
2170                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2171                 return (err);
2172         }
2173
2174         /* Register this device to handle MSI interrupts */
2175         err = intr_msi_register(dev, xref);
2176         if (err != 0) {
2177                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2178                 return (err);
2179         }
2180
2181         return (0);
2182 }
2183 #endif
2184
2185 #ifdef DEV_ACPI
2186 static device_probe_t gicv3_its_acpi_probe;
2187 static device_attach_t gicv3_its_acpi_attach;
2188
2189 static device_method_t gicv3_its_acpi_methods[] = {
2190         /* Device interface */
2191         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
2192         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
2193
2194         /* End */
2195         DEVMETHOD_END
2196 };
2197
2198 #define its_baseclasses its_acpi_baseclasses
2199 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2200     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2201 #undef its_baseclasses
2202
2203 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2204     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2205
2206 static int
2207 gicv3_its_acpi_probe(device_t dev)
2208 {
2209
2210         if (gic_get_bus(dev) != GIC_BUS_ACPI)
2211                 return (EINVAL);
2212
2213         if (gic_get_hw_rev(dev) < 3)
2214                 return (EINVAL);
2215
2216         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2217         return (BUS_PROBE_DEFAULT);
2218 }
2219
2220 static int
2221 gicv3_its_acpi_attach(device_t dev)
2222 {
2223         struct gicv3_its_softc *sc;
2224         struct gic_v3_devinfo *di;
2225         int err;
2226
2227         sc = device_get_softc(dev);
2228         sc->dev = dev;
2229         err = gicv3_its_attach(dev);
2230         if (err != 0)
2231                 return (err);
2232
2233         di = device_get_ivars(dev);
2234         sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2235         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2236             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2237         if (err != 0) {
2238                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2239                 return (err);
2240         }
2241
2242         /* Register this device to handle MSI interrupts */
2243         err = intr_msi_register(dev, di->msi_xref);
2244         if (err != 0) {
2245                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2246                 return (err);
2247         }
2248
2249         return (0);
2250 }
2251 #endif