sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  *
   4  * This software was developed by Andrew Turner under
   5  * the sponsorship of the FreeBSD Foundation.
   6  *
   7  * This software was developed by Semihalf under
   8  * the sponsorship of the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include "opt_acpi.h"
  33 #include "opt_platform.h"
  34 #include "opt_iommu.h"
  35
  36 #include <sys/cdefs.h>
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/bus.h>
  40 #include <sys/cpuset.h>
  41 #include <sys/endian.h>
  42 #include <sys/kernel.h>
  43 #include <sys/lock.h>
  44 #include <sys/malloc.h>
  45 #include <sys/module.h>
  46 #include <sys/mutex.h>
  47 #include <sys/proc.h>
  48 #include <sys/taskqueue.h>
  49 #include <sys/tree.h>
  50 #include <sys/queue.h>
  51 #include <sys/rman.h>
  52 #include <sys/sbuf.h>
  53 #include <sys/smp.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/vmem.h>
  56
  57 #include <vm/vm.h>
  58 #include <vm/pmap.h>
  59 #include <vm/vm_page.h>
  60
  61 #include <machine/bus.h>
  62 #include <machine/intr.h>
  63
  64 #include <arm/arm/gic_common.h>
  65 #include <arm64/arm64/gic_v3_reg.h>
  66 #include <arm64/arm64/gic_v3_var.h>
  67
  68 #ifdef FDT
  69 #include <dev/ofw/openfirm.h>
  70 #include <dev/ofw/ofw_bus.h>
  71 #include <dev/ofw/ofw_bus_subr.h>
  72 #endif
  73 #include <dev/pci/pcireg.h>
  74 #include <dev/pci/pcivar.h>
  75
  76 #ifdef IOMMU
  77 #include <dev/iommu/iommu.h>
  78 #include <dev/iommu/iommu_gas.h>
  79 #endif
  80
  81 #include "pcib_if.h"
  82 #include "pic_if.h"
  83 #include "msi_if.h"
  84
  85 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  86     "ARM GICv3 Interrupt Translation Service");
  87
  88 #define LPI_NIRQS               (64 * 1024)
  89
  90 /* The size and alignment of the command circular buffer */
  91 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  92 #define ITS_CMDQ_ALIGN          (64 * 1024)
  93
  94 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  95 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  96 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  97
  98 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
  99 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
 100 #define LPI_PENDTAB_ALIGN       (64 * 1024)
 101 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 102
 103 #define LPI_INT_TRANS_TAB_ALIGN 256
 104 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
 105
 106 /* ITS commands encoding */
 107 #define ITS_CMD_MOVI            (0x01)
 108 #define ITS_CMD_SYNC            (0x05)
 109 #define ITS_CMD_MAPD            (0x08)
 110 #define ITS_CMD_MAPC            (0x09)
 111 #define ITS_CMD_MAPTI           (0x0a)
 112 #define ITS_CMD_MAPI            (0x0b)
 113 #define ITS_CMD_INV             (0x0c)
 114 #define ITS_CMD_INVALL          (0x0d)
 115 /* Command */
 116 #define CMD_COMMAND_MASK        (0xFFUL)
 117 /* PCI device ID */
 118 #define CMD_DEVID_SHIFT         (32)
 119 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 120 /* Size of IRQ ID bitfield */
 121 #define CMD_SIZE_MASK           (0xFFUL)
 122 /* Virtual LPI ID */
 123 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 124 /* Physical LPI ID */
 125 #define CMD_PID_SHIFT           (32)
 126 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 127 /* Collection */
 128 #define CMD_COL_MASK            (0xFFFFUL)
 129 /* Target (CPU or Re-Distributor) */
 130 #define CMD_TARGET_SHIFT        (16)
 131 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 132 /* Interrupt Translation Table address */
 133 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 134 /* Valid command bit */
 135 #define CMD_VALID_SHIFT         (63)
 136 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 137
 138 #define ITS_TARGET_NONE         0xFBADBEEF
 139
 140 /* LPI chunk owned by ITS device */
 141 struct lpi_chunk {
 142         u_int   lpi_base;
 143         u_int   lpi_free;       /* First free LPI in set */
 144         u_int   lpi_num;        /* Total number of LPIs in chunk */
 145         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 146 };
 147
 148 /* ITS device */
 149 struct its_dev {
 150         TAILQ_ENTRY(its_dev)    entry;
 151         /* PCI device */
 152         device_t                pci_dev;
 153         /* Device ID (i.e. PCI device ID) */
 154         uint32_t                devid;
 155         /* List of assigned LPIs */
 156         struct lpi_chunk        lpis;
 157         /* Virtual address of ITT */
 158         vm_offset_t             itt;
 159         size_t                  itt_size;
 160 };
 161
 162 /*
 163  * ITS command descriptor.
 164  * Idea for command description passing taken from Linux.
 165  */
 166 struct its_cmd_desc {
 167         uint8_t cmd_type;
 168
 169         union {
 170                 struct {
 171                         struct its_dev *its_dev;
 172                         struct its_col *col;
 173                         uint32_t id;
 174                 } cmd_desc_movi;
 175
 176                 struct {
 177                         struct its_col *col;
 178                 } cmd_desc_sync;
 179
 180                 struct {
 181                         struct its_col *col;
 182                         uint8_t valid;
 183                 } cmd_desc_mapc;
 184
 185                 struct {
 186                         struct its_dev *its_dev;
 187                         struct its_col *col;
 188                         uint32_t pid;
 189                         uint32_t id;
 190                 } cmd_desc_mapvi;
 191
 192                 struct {
 193                         struct its_dev *its_dev;
 194                         struct its_col *col;
 195                         uint32_t pid;
 196                 } cmd_desc_mapi;
 197
 198                 struct {
 199                         struct its_dev *its_dev;
 200                         uint8_t valid;
 201                 } cmd_desc_mapd;
 202
 203                 struct {
 204                         struct its_dev *its_dev;
 205                         struct its_col *col;
 206                         uint32_t pid;
 207                 } cmd_desc_inv;
 208
 209                 struct {
 210                         struct its_col *col;
 211                 } cmd_desc_invall;
 212         };
 213 };
 214
 215 /* ITS command. Each command is 32 bytes long */
 216 struct its_cmd {
 217         uint64_t        cmd_dword[4];   /* ITS command double word */
 218 };
 219
 220 /* An ITS private table */
 221 struct its_ptable {
 222         vm_offset_t     ptab_vaddr;
 223         unsigned long   ptab_size;
 224 };
 225
 226 /* ITS collection description. */
 227 struct its_col {
 228         uint64_t        col_target;     /* Target Re-Distributor */
 229         uint64_t        col_id;         /* Collection ID */
 230 };
 231
 232 struct gicv3_its_irqsrc {
 233         struct intr_irqsrc      gi_isrc;
 234         u_int                   gi_id;
 235         u_int                   gi_lpi;
 236         struct its_dev          *gi_its_dev;
 237         TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
 238 };
 239
 240 struct gicv3_its_softc {
 241         device_t        dev;
 242         struct intr_pic *sc_pic;
 243         struct resource *sc_its_res;
 244
 245         cpuset_t        sc_cpus;
 246         u_int           gic_irq_cpu;
 247
 248         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 249         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 250
 251         /*
 252          * TODO: We should get these from the parent as we only want a
 253          * single copy of each across the interrupt controller.
 254          */
 255         uint8_t         *sc_conf_base;
 256         vm_offset_t sc_pend_base[MAXCPU];
 257
 258         /* Command handling */
 259         struct mtx sc_its_cmd_lock;
 260         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 261         size_t sc_its_cmd_next_idx;
 262
 263         vmem_t *sc_irq_alloc;
 264         struct gicv3_its_irqsrc **sc_irqs;
 265         u_int   sc_irq_base;
 266         u_int   sc_irq_length;
 267         u_int   sc_irq_count;
 268
 269         struct mtx sc_its_dev_lock;
 270         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 271         TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
 272
 273 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 274 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 275 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 276         u_int sc_its_flags;
 277         bool    trace_enable;
 278         vm_page_t ma; /* fake msi page */
 279 };
 280
 281 static void *conf_base;
 282
 283 typedef void (its_quirk_func_t)(device_t);
 284 static its_quirk_func_t its_quirk_cavium_22375;
 285
 286 static const struct {
 287         const char *desc;
 288         uint32_t iidr;
 289         uint32_t iidr_mask;
 290         its_quirk_func_t *func;
 291 } its_quirks[] = {
 292         {
 293                 /* Cavium ThunderX Pass 1.x */
 294                 .desc = "Cavium ThunderX errata: 22375, 24313",
 295                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 296                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 297                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 298                 .func = its_quirk_cavium_22375,
 299         },
 300 };
 301
 302 #define gic_its_read_4(sc, reg)                 \
 303     bus_read_4((sc)->sc_its_res, (reg))
 304 #define gic_its_read_8(sc, reg)                 \
 305     bus_read_8((sc)->sc_its_res, (reg))
 306
 307 #define gic_its_write_4(sc, reg, val)           \
 308     bus_write_4((sc)->sc_its_res, (reg), (val))
 309 #define gic_its_write_8(sc, reg, val)           \
 310     bus_write_8((sc)->sc_its_res, (reg), (val))
 311
 312 static device_attach_t gicv3_its_attach;
 313 static device_detach_t gicv3_its_detach;
 314
 315 static pic_disable_intr_t gicv3_its_disable_intr;
 316 static pic_enable_intr_t gicv3_its_enable_intr;
 317 static pic_map_intr_t gicv3_its_map_intr;
 318 static pic_setup_intr_t gicv3_its_setup_intr;
 319 static pic_post_filter_t gicv3_its_post_filter;
 320 static pic_post_ithread_t gicv3_its_post_ithread;
 321 static pic_pre_ithread_t gicv3_its_pre_ithread;
 322 static pic_bind_intr_t gicv3_its_bind_intr;
 323 #ifdef SMP
 324 static pic_init_secondary_t gicv3_its_init_secondary;
 325 #endif
 326 static msi_alloc_msi_t gicv3_its_alloc_msi;
 327 static msi_release_msi_t gicv3_its_release_msi;
 328 static msi_alloc_msix_t gicv3_its_alloc_msix;
 329 static msi_release_msix_t gicv3_its_release_msix;
 330 static msi_map_msi_t gicv3_its_map_msi;
 331 #ifdef IOMMU
 332 static msi_iommu_init_t gicv3_iommu_init;
 333 static msi_iommu_deinit_t gicv3_iommu_deinit;
 334 #endif
 335
 336 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 337 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 338 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 339 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 340 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 341 static void its_cmd_invall(device_t, struct its_col *);
 342
 343 static device_method_t gicv3_its_methods[] = {
 344         /* Device interface */
 345         DEVMETHOD(device_detach,        gicv3_its_detach),
 346
 347         /* Interrupt controller interface */
 348         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 349         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 350         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 351         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 352         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 353         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 354         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 355 #ifdef SMP
 356         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 357         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 358 #endif
 359
 360         /* MSI/MSI-X */
 361         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 362         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 363         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 364         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 365         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 366 #ifdef IOMMU
 367         DEVMETHOD(msi_iommu_init,       gicv3_iommu_init),
 368         DEVMETHOD(msi_iommu_deinit,     gicv3_iommu_deinit),
 369 #endif
 370
 371         /* End */
 372         DEVMETHOD_END
 373 };
 374
 375 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 376     sizeof(struct gicv3_its_softc));
 377
 378 static void
 379 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 380 {
 381         vm_paddr_t cmd_paddr;
 382         uint64_t reg, tmp;
 383
 384         /* Set up the command circular buffer */
 385         sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS,
 386             M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0);
 387         sc->sc_its_cmd_next_idx = 0;
 388
 389         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 390
 391         /* Set the base of the command buffer */
 392         reg = GITS_CBASER_VALID |
 393             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 394             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 395             (ITS_CMDQ_SIZE / 4096 - 1);
 396         gic_its_write_8(sc, GITS_CBASER, reg);
 397
 398         /* Read back to check for fixed value fields */
 399         tmp = gic_its_read_8(sc, GITS_CBASER);
 400
 401         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 402             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 403                 /* Check if the hardware reported non-shareable */
 404                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 405                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 406                         /* If so remove the cache attribute */
 407                         reg &= ~GITS_CBASER_CACHE_MASK;
 408                         reg &= ~GITS_CBASER_SHARE_MASK;
 409                         /* Set to Non-cacheable, Non-shareable */
 410                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 411                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 412
 413                         gic_its_write_8(sc, GITS_CBASER, reg);
 414                 }
 415
 416                 /* The command queue has to be flushed after each command */
 417                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 418         }
 419
 420         /* Get the next command from the start of the buffer */
 421         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 422 }
 423
 424 static int
 425 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
 426 {
 427         uint64_t reg, tmp;
 428         int page_size;
 429
 430         page_size = PAGE_SIZE_64K;
 431         reg = gic_its_read_8(sc, GITS_BASER(table));
 432
 433         while (1) {
 434                 reg &= GITS_BASER_PSZ_MASK;
 435                 switch (page_size) {
 436                 case PAGE_SIZE_4K:      /* 4KB */
 437                         reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 438                         break;
 439                 case PAGE_SIZE_16K:     /* 16KB */
 440                         reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 441                         break;
 442                 case PAGE_SIZE_64K:     /* 64KB */
 443                         reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 444                         break;
 445                 }
 446
 447                 /* Write the new page size */
 448                 gic_its_write_8(sc, GITS_BASER(table), reg);
 449
 450                 /* Read back to check */
 451                 tmp = gic_its_read_8(sc, GITS_BASER(table));
 452
 453                 /* The page size is correct */
 454                 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
 455                         return (page_size);
 456
 457                 switch (page_size) {
 458                 default:
 459                         return (-1);
 460                 case PAGE_SIZE_16K:
 461                         page_size = PAGE_SIZE_4K;
 462                         break;
 463                 case PAGE_SIZE_64K:
 464                         page_size = PAGE_SIZE_16K;
 465                         break;
 466                 }
 467         }
 468 }
 469
 470 static int
 471 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 472 {
 473         vm_offset_t table;
 474         vm_paddr_t paddr;
 475         uint64_t cache, reg, share, tmp, type;
 476         size_t esize, its_tbl_size, nidents, nitspages, npages;
 477         int i, page_size;
 478         int devbits;
 479
 480         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 481                 /*
 482                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 483                  * are to be 21 bits in length. The entry size of the ITS
 484                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 485                  * is supposed to be 8 bytes in length (for device table).
 486                  * Finally the page size that is to be used by ITS to access
 487                  * this table will be set to 64KB.
 488                  *
 489                  * This gives 0x200000 entries of size 0x8 bytes covered by
 490                  * 256 pages each of which 64KB in size. The number of pages
 491                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 492                  * that case this value would be 0xFF but on ThunderX the
 493                  * maximum value that HW accepts is 0xFD.
 494                  *
 495                  * Set an arbitrary number of device ID bits to 20 in order
 496                  * to limit the number of entries in ITS device table to
 497                  * 0x100000 and the table size to 8MB.
 498                  */
 499                 devbits = 20;
 500                 cache = 0;
 501         } else {
 502                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 503                 cache = GITS_BASER_CACHE_WAWB;
 504         }
 505         share = GITS_BASER_SHARE_IS;
 506
 507         for (i = 0; i < GITS_BASER_NUM; i++) {
 508                 reg = gic_its_read_8(sc, GITS_BASER(i));
 509                 /* The type of table */
 510                 type = GITS_BASER_TYPE(reg);
 511                 if (type == GITS_BASER_TYPE_UNIMPL)
 512                         continue;
 513
 514                 /* The table entry size */
 515                 esize = GITS_BASER_ESIZE(reg);
 516
 517                 /* Find the tables page size */
 518                 page_size = gicv3_its_table_page_size(sc, i);
 519                 if (page_size == -1) {
 520                         device_printf(dev, "No valid page size for table %d\n",
 521                             i);
 522                         return (EINVAL);
 523                 }
 524
 525                 switch(type) {
 526                 case GITS_BASER_TYPE_DEV:
 527                         nidents = (1 << devbits);
 528                         its_tbl_size = esize * nidents;
 529                         its_tbl_size = roundup2(its_tbl_size, page_size);
 530                         break;
 531                 case GITS_BASER_TYPE_VP:
 532                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 533                 case GITS_BASER_TYPE_IC:
 534                         its_tbl_size = page_size;
 535                         break;
 536                 default:
 537                         continue;
 538                 }
 539                 npages = howmany(its_tbl_size, PAGE_SIZE);
 540
 541                 /* Allocate the table */
 542                 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,
 543                     M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
 544                     PAGE_SIZE_64K, 0);
 545
 546                 sc->sc_its_ptab[i].ptab_vaddr = table;
 547                 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;
 548
 549                 paddr = vtophys(table);
 550
 551                 while (1) {
 552                         nitspages = howmany(its_tbl_size, page_size);
 553
 554                         /* Clear the fields we will be setting */
 555                         reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
 556                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 557                             GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK |
 558                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 559                             GITS_BASER_SIZE_MASK);
 560                         /* Set the new values */
 561                         reg |= GITS_BASER_VALID |
 562                             (cache << GITS_BASER_CACHE_SHIFT) |
 563                             (type << GITS_BASER_TYPE_SHIFT) |
 564                             ((esize - 1) << GITS_BASER_ESIZE_SHIFT) |
 565                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 566                             (nitspages - 1);
 567
 568                         switch (page_size) {
 569                         case PAGE_SIZE_4K:      /* 4KB */
 570                                 reg |=
 571                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 572                                 break;
 573                         case PAGE_SIZE_16K:     /* 16KB */
 574                                 reg |=
 575                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 576                                 break;
 577                         case PAGE_SIZE_64K:     /* 64KB */
 578                                 reg |=
 579                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 580                                 break;
 581                         }
 582
 583                         gic_its_write_8(sc, GITS_BASER(i), reg);
 584
 585                         /* Read back to check */
 586                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 587
 588                         /* Do the shareability masks line up? */
 589                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 590                             (reg & GITS_BASER_SHARE_MASK)) {
 591                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 592                                     GITS_BASER_SHARE_SHIFT;
 593                                 continue;
 594                         }
 595
 596                         if (tmp != reg) {
 597                                 device_printf(dev, "GITS_BASER%d: "
 598                                     "unable to be updated: %lx != %lx\n",
 599                                     i, reg, tmp);
 600                                 return (ENXIO);
 601                         }
 602
 603                         /* We should have made all needed changes */
 604                         break;
 605                 }
 606         }
 607
 608         return (0);
 609 }
 610
 611 static void
 612 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 613 {
 614         void *conf_table;
 615
 616         conf_table = atomic_load_ptr(&conf_base);
 617         if (conf_table == NULL) {
 618                 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
 619                     M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
 620                     LPI_CONFTAB_ALIGN, 0);
 621
 622                 if (atomic_cmpset_ptr((uintptr_t *)&conf_base,
 623                     (uintptr_t)NULL, (uintptr_t)conf_table) == 0) {
 624                         contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS);
 625                         conf_table = atomic_load_ptr(&conf_base);
 626                 }
 627         }
 628         sc->sc_conf_base = conf_table;
 629
 630         /* Set the default configuration */
 631         memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 632             LPI_CONFTAB_SIZE);
 633
 634         /* Flush the table to memory */
 635         cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
 636 }
 637
 638 static void
 639 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 640 {
 641         int i;
 642
 643         for (i = 0; i <= mp_maxid; i++) {
 644                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 645                         continue;
 646
 647                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 648                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 649                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 650
 651                 /* Flush so the ITS can see the memory */
 652                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
 653                     LPI_PENDTAB_SIZE);
 654         }
 655 }
 656
 657 static void
 658 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
 659 {
 660         device_t gicv3;
 661         uint64_t xbaser, tmp;
 662         uint32_t ctlr;
 663         u_int cpuid;
 664
 665         gicv3 = device_get_parent(dev);
 666         cpuid = PCPU_GET(cpuid);
 667
 668         /* Disable LPIs */
 669         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 670         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 671         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 672
 673         /* Make sure changes are observable my the GIC */
 674         dsb(sy);
 675
 676         /*
 677          * Set the redistributor base
 678          */
 679         xbaser = vtophys(sc->sc_conf_base) |
 680             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 681             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 682             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 683         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 684
 685         /* Check the cache attributes we set */
 686         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 687
 688         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 689             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 690                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 691                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 692                         /* We need to mark as non-cacheable */
 693                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 694                             GICR_PROPBASER_CACHE_MASK);
 695                         /* Non-cacheable */
 696                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 697                             GICR_PROPBASER_CACHE_SHIFT;
 698                         /* Non-sareable */
 699                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 700                             GICR_PROPBASER_SHARE_SHIFT;
 701                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 702                 }
 703                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 704         }
 705
 706         /*
 707          * Set the LPI pending table base
 708          */
 709         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 710             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 711             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 712
 713         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 714
 715         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 716
 717         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 718             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 719                 /* Clear the cahce and shareability bits */
 720                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 721                     GICR_PENDBASER_SHARE_MASK);
 722                 /* Mark as non-shareable */
 723                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 724                 /* And non-cacheable */
 725                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 726                     GICR_PENDBASER_CACHE_SHIFT;
 727         }
 728
 729         /* Enable LPIs */
 730         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 731         ctlr |= GICR_CTLR_LPI_ENABLE;
 732         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 733
 734         /* Make sure the GIC has seen everything */
 735         dsb(sy);
 736 }
 737
 738 static int
 739 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 740 {
 741         device_t gicv3;
 742         vm_paddr_t target;
 743         u_int cpuid;
 744         struct redist_pcpu *rpcpu;
 745
 746         gicv3 = device_get_parent(dev);
 747         cpuid = PCPU_GET(cpuid);
 748         if (!CPU_ISSET(cpuid, &sc->sc_cpus))
 749                 return (0);
 750
 751         /* Check if the ITS is enabled on this CPU */
 752         if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
 753                 return (ENXIO);
 754
 755         rpcpu = gicv3_get_redist(dev);
 756
 757         /* Do per-cpu LPI init once */
 758         if (!rpcpu->lpi_enabled) {
 759                 its_init_cpu_lpi(dev, sc);
 760                 rpcpu->lpi_enabled = true;
 761         }
 762
 763         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 764                 /* This ITS wants the redistributor physical address */
 765                 target = vtophys(rman_get_virtual(&rpcpu->res));
 766         } else {
 767                 /* This ITS wants the unique processor number */
 768                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
 769                     CMD_TARGET_SHIFT;
 770         }
 771
 772         sc->sc_its_cols[cpuid]->col_target = target;
 773         sc->sc_its_cols[cpuid]->col_id = cpuid;
 774
 775         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 776         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 777
 778         return (0);
 779 }
 780
 781 static int
 782 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
 783 {
 784         struct gicv3_its_softc *sc;
 785         int rv;
 786
 787         sc = arg1;
 788
 789         rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
 790         if (rv != 0 || req->newptr == NULL)
 791                 return (rv);
 792         if (sc->trace_enable)
 793                 gic_its_write_8(sc, GITS_TRKCTLR, 3);
 794         else
 795                 gic_its_write_8(sc, GITS_TRKCTLR, 0);
 796
 797         return (0);
 798 }
 799
 800 static int
 801 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
 802 {
 803         struct gicv3_its_softc *sc;
 804         struct sbuf *sb;
 805         int err;
 806
 807         sc = arg1;
 808         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 809         if (sb == NULL) {
 810                 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
 811                 return (ENOMEM);
 812         }
 813         sbuf_cat(sb, "\n");
 814         sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
 815             gic_its_read_4(sc, GITS_TRKCTLR));
 816         sbuf_printf(sb, "GITS_TRKR:    0x%08X\n",
 817             gic_its_read_4(sc, GITS_TRKR));
 818         sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
 819             gic_its_read_4(sc, GITS_TRKDIDR));
 820         sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
 821             gic_its_read_4(sc, GITS_TRKPIDR));
 822         sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
 823             gic_its_read_4(sc, GITS_TRKVIDR));
 824         sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
 825            gic_its_read_4(sc, GITS_TRKTGTR));
 826
 827         err = sbuf_finish(sb);
 828         if (err)
 829                 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
 830         sbuf_delete(sb);
 831         return(err);
 832 }
 833
 834 static int
 835 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
 836 {
 837         struct sysctl_oid *oid, *child;
 838         struct sysctl_ctx_list *ctx_list;
 839
 840         ctx_list = device_get_sysctl_ctx(sc->dev);
 841         child = device_get_sysctl_tree(sc->dev);
 842         oid = SYSCTL_ADD_NODE(ctx_list,
 843             SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
 844             CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
 845         if (oid == NULL)
 846                 return (ENXIO);
 847
 848         /* Add registers */
 849         SYSCTL_ADD_PROC(ctx_list,
 850             SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
 851             CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 852             gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
 853         SYSCTL_ADD_PROC(ctx_list,
 854             SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
 855             CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 856             gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
 857
 858         return (0);
 859 }
 860
 861 static int
 862 gicv3_its_attach(device_t dev)
 863 {
 864         struct gicv3_its_softc *sc;
 865         int domain, err, i, rid;
 866         uint64_t phys;
 867         uint32_t ctlr, iidr;
 868
 869         sc = device_get_softc(dev);
 870
 871         sc->sc_irq_length = gicv3_get_nirqs(dev);
 872         sc->sc_irq_base = GIC_FIRST_LPI;
 873         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
 874
 875         rid = 0;
 876         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 877             RF_ACTIVE);
 878         if (sc->sc_its_res == NULL) {
 879                 device_printf(dev, "Could not allocate memory\n");
 880                 return (ENXIO);
 881         }
 882
 883         phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
 884             GITS_TRANSLATER, PAGE_SIZE);
 885         sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
 886         vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
 887
 888         iidr = gic_its_read_4(sc, GITS_IIDR);
 889         for (i = 0; i < nitems(its_quirks); i++) {
 890                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 891                         if (bootverbose) {
 892                                 device_printf(dev, "Applying %s\n",
 893                                     its_quirks[i].desc);
 894                         }
 895                         its_quirks[i].func(dev);
 896                         break;
 897                 }
 898         }
 899
 900         /*
 901          * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
 902          * coming in via, for instance, a kexec/kboot style setup where a
 903          * previous kernel has configured then relinquished control.  Clear it
 904          * so that we can reconfigure GITS_BASER*.
 905          */
 906         ctlr = gic_its_read_4(sc, GITS_CTLR);
 907         if ((ctlr & GITS_CTLR_EN) != 0) {
 908                 ctlr &= ~GITS_CTLR_EN;
 909                 gic_its_write_4(sc, GITS_CTLR, ctlr);
 910         }
 911
 912         /* Allocate the private tables */
 913         err = gicv3_its_table_init(dev, sc);
 914         if (err != 0)
 915                 return (err);
 916
 917         /* Protects access to the device list */
 918         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 919
 920         /* Protects access to the ITS command circular buffer. */
 921         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
 922
 923         CPU_ZERO(&sc->sc_cpus);
 924         if (bus_get_domain(dev, &domain) == 0) {
 925                 if (domain < MAXMEMDOM)
 926                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
 927         } else {
 928                 CPU_COPY(&all_cpus, &sc->sc_cpus);
 929         }
 930
 931         /* Allocate the command circular buffer */
 932         gicv3_its_cmdq_init(sc);
 933
 934         /* Allocate the per-CPU collections */
 935         for (int cpu = 0; cpu <= mp_maxid; cpu++)
 936                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
 937                         sc->sc_its_cols[cpu] = malloc(
 938                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
 939                             M_WAITOK | M_ZERO);
 940                 else
 941                         sc->sc_its_cols[cpu] = NULL;
 942
 943         /* Enable the ITS */
 944         gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
 945
 946         /* Create the LPI configuration table */
 947         gicv3_its_conftable_init(sc);
 948
 949         /* And the pending tebles */
 950         gicv3_its_pendtables_init(sc);
 951
 952         /* Enable LPIs on this CPU */
 953         its_init_cpu(dev, sc);
 954
 955         TAILQ_INIT(&sc->sc_its_dev_list);
 956         TAILQ_INIT(&sc->sc_free_irqs);
 957
 958         /*
 959          * Create the vmem object to allocate INTRNG IRQs from. We try to
 960          * use all IRQs not already used by the GICv3.
 961          * XXX: This assumes there are no other interrupt controllers in the
 962          * system.
 963          */
 964         sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
 965             gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
 966
 967         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
 968             M_GICV3_ITS, M_WAITOK | M_ZERO);
 969
 970         /* For GIC-500 install tracking sysctls. */
 971         if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
 972             GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
 973                 gicv3_its_init_sysctl(sc);
 974
 975         return (0);
 976 }
 977
 978 static int
 979 gicv3_its_detach(device_t dev)
 980 {
 981
 982         return (ENXIO);
 983 }
 984
 985 static void
 986 its_quirk_cavium_22375(device_t dev)
 987 {
 988         struct gicv3_its_softc *sc;
 989
 990         sc = device_get_softc(dev);
 991         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
 992 }
 993
 994 static void
 995 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
 996 {
 997         struct gicv3_its_softc *sc;
 998         struct gicv3_its_irqsrc *girq;
 999         uint8_t *conf;
1000
1001         sc = device_get_softc(dev);
1002         girq = (struct gicv3_its_irqsrc *)isrc;
1003         conf = sc->sc_conf_base;
1004
1005         conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1006
1007         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1008                 /* Clean D-cache under command. */
1009                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1010         } else {
1011                 /* DSB inner shareable, store */
1012                 dsb(ishst);
1013         }
1014
1015         its_cmd_inv(dev, girq->gi_its_dev, girq);
1016 }
1017
1018 static void
1019 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1020 {
1021         struct gicv3_its_softc *sc;
1022         struct gicv3_its_irqsrc *girq;
1023         uint8_t *conf;
1024
1025         sc = device_get_softc(dev);
1026         girq = (struct gicv3_its_irqsrc *)isrc;
1027         conf = sc->sc_conf_base;
1028
1029         conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1030
1031         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1032                 /* Clean D-cache under command. */
1033                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1034         } else {
1035                 /* DSB inner shareable, store */
1036                 dsb(ishst);
1037         }
1038
1039         its_cmd_inv(dev, girq->gi_its_dev, girq);
1040 }
1041
1042 static int
1043 gicv3_its_intr(void *arg, uintptr_t irq)
1044 {
1045         struct gicv3_its_softc *sc = arg;
1046         struct gicv3_its_irqsrc *girq;
1047         struct trapframe *tf;
1048
1049         irq -= sc->sc_irq_base;
1050         girq = sc->sc_irqs[irq];
1051         if (girq == NULL)
1052                 panic("gicv3_its_intr: Invalid interrupt %ld",
1053                     irq + sc->sc_irq_base);
1054
1055         tf = curthread->td_intr_frame;
1056         intr_isrc_dispatch(&girq->gi_isrc, tf);
1057         return (FILTER_HANDLED);
1058 }
1059
1060 static void
1061 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1062 {
1063         struct gicv3_its_irqsrc *girq;
1064         struct gicv3_its_softc *sc;
1065
1066         sc = device_get_softc(dev);
1067         girq = (struct gicv3_its_irqsrc *)isrc;
1068         gicv3_its_disable_intr(dev, isrc);
1069         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1070 }
1071
1072 static void
1073 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1074 {
1075
1076         gicv3_its_enable_intr(dev, isrc);
1077 }
1078
1079 static void
1080 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1081 {
1082         struct gicv3_its_irqsrc *girq;
1083         struct gicv3_its_softc *sc;
1084
1085         sc = device_get_softc(dev);
1086         girq = (struct gicv3_its_irqsrc *)isrc;
1087         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1088 }
1089
1090 static int
1091 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1092 {
1093         struct gicv3_its_softc *sc;
1094
1095         sc = device_get_softc(dev);
1096         if (CPU_EMPTY(&isrc->isrc_cpu)) {
1097                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1098                     &sc->sc_cpus);
1099                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1100         }
1101
1102         return (0);
1103 }
1104
1105 static int
1106 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1107 {
1108         struct gicv3_its_irqsrc *girq;
1109
1110         gicv3_its_select_cpu(dev, isrc);
1111
1112         girq = (struct gicv3_its_irqsrc *)isrc;
1113         its_cmd_movi(dev, girq);
1114         return (0);
1115 }
1116
1117 static int
1118 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1119     struct intr_irqsrc **isrcp)
1120 {
1121
1122         /*
1123          * This should never happen, we only call this function to map
1124          * interrupts found before the controller driver is ready.
1125          */
1126         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1127 }
1128
1129 static int
1130 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1131     struct resource *res, struct intr_map_data *data)
1132 {
1133
1134         /* Bind the interrupt to a CPU */
1135         gicv3_its_bind_intr(dev, isrc);
1136
1137         return (0);
1138 }
1139
1140 #ifdef SMP
1141 static void
1142 gicv3_its_init_secondary(device_t dev)
1143 {
1144         struct gicv3_its_softc *sc;
1145
1146         sc = device_get_softc(dev);
1147
1148         /*
1149          * This is fatal as otherwise we may bind interrupts to this CPU.
1150          * We need a way to tell the interrupt framework to only bind to a
1151          * subset of given CPUs when it performs the shuffle.
1152          */
1153         if (its_init_cpu(dev, sc) != 0)
1154                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1155                     PCPU_GET(cpuid));
1156 }
1157 #endif
1158
1159 static uint32_t
1160 its_get_devid(device_t pci_dev)
1161 {
1162         uintptr_t id;
1163
1164         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1165                 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1166                     device_get_nameunit(pci_dev));
1167
1168         return (id);
1169 }
1170
1171 static struct its_dev *
1172 its_device_find(device_t dev, device_t child)
1173 {
1174         struct gicv3_its_softc *sc;
1175         struct its_dev *its_dev = NULL;
1176
1177         sc = device_get_softc(dev);
1178
1179         mtx_lock_spin(&sc->sc_its_dev_lock);
1180         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1181                 if (its_dev->pci_dev == child)
1182                         break;
1183         }
1184         mtx_unlock_spin(&sc->sc_its_dev_lock);
1185
1186         return (its_dev);
1187 }
1188
1189 static struct its_dev *
1190 its_device_get(device_t dev, device_t child, u_int nvecs)
1191 {
1192         struct gicv3_its_softc *sc;
1193         struct its_dev *its_dev;
1194         vmem_addr_t irq_base;
1195         size_t esize;
1196
1197         sc = device_get_softc(dev);
1198
1199         its_dev = its_device_find(dev, child);
1200         if (its_dev != NULL)
1201                 return (its_dev);
1202
1203         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1204         if (its_dev == NULL)
1205                 return (NULL);
1206
1207         its_dev->pci_dev = child;
1208         its_dev->devid = its_get_devid(child);
1209
1210         its_dev->lpis.lpi_busy = 0;
1211         its_dev->lpis.lpi_num = nvecs;
1212         its_dev->lpis.lpi_free = nvecs;
1213
1214         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1215             &irq_base) != 0) {
1216                 free(its_dev, M_GICV3_ITS);
1217                 return (NULL);
1218         }
1219         its_dev->lpis.lpi_base = irq_base;
1220
1221         /* Get ITT entry size */
1222         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1223
1224         /*
1225          * Allocate ITT for this device.
1226          * PA has to be 256 B aligned. At least two entries for device.
1227          */
1228         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1229         its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size,
1230             M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR,
1231             LPI_INT_TRANS_TAB_ALIGN, 0);
1232         if (its_dev->itt == 0) {
1233                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1234                 free(its_dev, M_GICV3_ITS);
1235                 return (NULL);
1236         }
1237
1238         mtx_lock_spin(&sc->sc_its_dev_lock);
1239         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1240         mtx_unlock_spin(&sc->sc_its_dev_lock);
1241
1242         /* Map device to its ITT */
1243         its_cmd_mapd(dev, its_dev, 1);
1244
1245         return (its_dev);
1246 }
1247
1248 static void
1249 its_device_release(device_t dev, struct its_dev *its_dev)
1250 {
1251         struct gicv3_its_softc *sc;
1252
1253         KASSERT(its_dev->lpis.lpi_busy == 0,
1254             ("its_device_release: Trying to release an inuse ITS device"));
1255
1256         /* Unmap device in ITS */
1257         its_cmd_mapd(dev, its_dev, 0);
1258
1259         sc = device_get_softc(dev);
1260
1261         /* Remove the device from the list of devices */
1262         mtx_lock_spin(&sc->sc_its_dev_lock);
1263         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1264         mtx_unlock_spin(&sc->sc_its_dev_lock);
1265
1266         /* Free ITT */
1267         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1268         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1269
1270         /* Free the IRQ allocation */
1271         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1272             its_dev->lpis.lpi_num);
1273
1274         free(its_dev, M_GICV3_ITS);
1275 }
1276
1277 static struct gicv3_its_irqsrc *
1278 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1279 {
1280         struct gicv3_its_irqsrc *girq = NULL;
1281
1282         KASSERT(sc->sc_irqs[irq] == NULL,
1283             ("%s: Interrupt %u already allocated", __func__, irq));
1284         mtx_lock_spin(&sc->sc_its_dev_lock);
1285         if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1286                 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1287                 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1288         }
1289         mtx_unlock_spin(&sc->sc_its_dev_lock);
1290         if (girq == NULL) {
1291                 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1292                     M_NOWAIT | M_ZERO);
1293                 if (girq == NULL)
1294                         return (NULL);
1295                 girq->gi_id = -1;
1296                 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1297                     "%s,%u", device_get_nameunit(dev), irq) != 0) {
1298                         free(girq, M_GICV3_ITS);
1299                         return (NULL);
1300                 }
1301         }
1302         girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1303         sc->sc_irqs[irq] = girq;
1304
1305         return (girq);
1306 }
1307
1308 static void
1309 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1310     struct gicv3_its_irqsrc *girq)
1311 {
1312         u_int irq;
1313
1314         mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1315
1316         irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1317         sc->sc_irqs[irq] = NULL;
1318
1319         girq->gi_id = -1;
1320         girq->gi_its_dev = NULL;
1321         TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1322 }
1323
1324 static int
1325 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1326     device_t *pic, struct intr_irqsrc **srcs)
1327 {
1328         struct gicv3_its_softc *sc;
1329         struct gicv3_its_irqsrc *girq;
1330         struct its_dev *its_dev;
1331         u_int irq;
1332         int i;
1333
1334         its_dev = its_device_get(dev, child, count);
1335         if (its_dev == NULL)
1336                 return (ENXIO);
1337
1338         KASSERT(its_dev->lpis.lpi_free >= count,
1339             ("gicv3_its_alloc_msi: No free LPIs"));
1340         sc = device_get_softc(dev);
1341         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1342             its_dev->lpis.lpi_free;
1343
1344         /* Allocate the irqsrc for each MSI */
1345         for (i = 0; i < count; i++, irq++) {
1346                 its_dev->lpis.lpi_free--;
1347                 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1348                     sc, irq);
1349                 if (srcs[i] == NULL)
1350                         break;
1351         }
1352
1353         /* The allocation failed, release them */
1354         if (i != count) {
1355                 mtx_lock_spin(&sc->sc_its_dev_lock);
1356                 for (i = 0; i < count; i++) {
1357                         girq = (struct gicv3_its_irqsrc *)srcs[i];
1358                         if (girq == NULL)
1359                                 break;
1360                         gicv3_its_release_irqsrc(sc, girq);
1361                         srcs[i] = NULL;
1362                 }
1363                 mtx_unlock_spin(&sc->sc_its_dev_lock);
1364                 return (ENXIO);
1365         }
1366
1367         /* Finish the allocation now we have all MSI irqsrcs */
1368         for (i = 0; i < count; i++) {
1369                 girq = (struct gicv3_its_irqsrc *)srcs[i];
1370                 girq->gi_id = i;
1371                 girq->gi_its_dev = its_dev;
1372
1373                 /* Map the message to the given IRQ */
1374                 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1375                 its_cmd_mapti(dev, girq);
1376         }
1377         its_dev->lpis.lpi_busy += count;
1378         *pic = dev;
1379
1380         return (0);
1381 }
1382
1383 static int
1384 gicv3_its_release_msi(device_t dev, device_t child, int count,
1385     struct intr_irqsrc **isrc)
1386 {
1387         struct gicv3_its_softc *sc;
1388         struct gicv3_its_irqsrc *girq;
1389         struct its_dev *its_dev;
1390         int i;
1391
1392         its_dev = its_device_find(dev, child);
1393
1394         KASSERT(its_dev != NULL,
1395             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1396              "no ITS device"));
1397         KASSERT(its_dev->lpis.lpi_busy >= count,
1398             ("gicv3_its_release_msi: Releasing more interrupts than "
1399              "were allocated: releasing %d, allocated %d", count,
1400              its_dev->lpis.lpi_busy));
1401
1402         sc = device_get_softc(dev);
1403         mtx_lock_spin(&sc->sc_its_dev_lock);
1404         for (i = 0; i < count; i++) {
1405                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1406                 gicv3_its_release_irqsrc(sc, girq);
1407         }
1408         mtx_unlock_spin(&sc->sc_its_dev_lock);
1409         its_dev->lpis.lpi_busy -= count;
1410
1411         if (its_dev->lpis.lpi_busy == 0)
1412                 its_device_release(dev, its_dev);
1413
1414         return (0);
1415 }
1416
1417 static int
1418 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1419     struct intr_irqsrc **isrcp)
1420 {
1421         struct gicv3_its_softc *sc;
1422         struct gicv3_its_irqsrc *girq;
1423         struct its_dev *its_dev;
1424         u_int nvecs, irq;
1425
1426         nvecs = pci_msix_count(child);
1427         its_dev = its_device_get(dev, child, nvecs);
1428         if (its_dev == NULL)
1429                 return (ENXIO);
1430
1431         KASSERT(its_dev->lpis.lpi_free > 0,
1432             ("gicv3_its_alloc_msix: No free LPIs"));
1433         sc = device_get_softc(dev);
1434         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1435             its_dev->lpis.lpi_free;
1436
1437         girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1438         if (girq == NULL)
1439                 return (ENXIO);
1440         girq->gi_id = its_dev->lpis.lpi_busy;
1441         girq->gi_its_dev = its_dev;
1442
1443         its_dev->lpis.lpi_free--;
1444         its_dev->lpis.lpi_busy++;
1445
1446         /* Map the message to the given IRQ */
1447         gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1448         its_cmd_mapti(dev, girq);
1449
1450         *pic = dev;
1451         *isrcp = (struct intr_irqsrc *)girq;
1452
1453         return (0);
1454 }
1455
1456 static int
1457 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1458 {
1459         struct gicv3_its_softc *sc;
1460         struct gicv3_its_irqsrc *girq;
1461         struct its_dev *its_dev;
1462
1463         its_dev = its_device_find(dev, child);
1464
1465         KASSERT(its_dev != NULL,
1466             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1467              "no ITS device"));
1468         KASSERT(its_dev->lpis.lpi_busy > 0,
1469             ("gicv3_its_release_msix: Releasing more interrupts than "
1470              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1471
1472         sc = device_get_softc(dev);
1473         girq = (struct gicv3_its_irqsrc *)isrc;
1474         mtx_lock_spin(&sc->sc_its_dev_lock);
1475         gicv3_its_release_irqsrc(sc, girq);
1476         mtx_unlock_spin(&sc->sc_its_dev_lock);
1477         its_dev->lpis.lpi_busy--;
1478
1479         if (its_dev->lpis.lpi_busy == 0)
1480                 its_device_release(dev, its_dev);
1481
1482         return (0);
1483 }
1484
1485 static int
1486 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1487     uint64_t *addr, uint32_t *data)
1488 {
1489         struct gicv3_its_softc *sc;
1490         struct gicv3_its_irqsrc *girq;
1491
1492         sc = device_get_softc(dev);
1493         girq = (struct gicv3_its_irqsrc *)isrc;
1494
1495         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1496         *data = girq->gi_id;
1497
1498         return (0);
1499 }
1500
1501 #ifdef IOMMU
1502 static int
1503 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1504 {
1505         struct gicv3_its_softc *sc;
1506         struct iommu_ctx *ctx;
1507         int error;
1508
1509         sc = device_get_softc(dev);
1510         ctx = iommu_get_dev_ctx(child);
1511         if (ctx == NULL)
1512                 return (ENXIO);
1513         /* Map the page containing the GITS_TRANSLATER register. */
1514         error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1515             IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1516         *domain = iommu_get_ctx_domain(ctx);
1517
1518         return (error);
1519 }
1520
1521 static void
1522 gicv3_iommu_deinit(device_t dev, device_t child)
1523 {
1524         struct iommu_ctx *ctx;
1525
1526         ctx = iommu_get_dev_ctx(child);
1527         if (ctx == NULL)
1528                 return;
1529
1530         iommu_unmap_msi(ctx);
1531 }
1532 #endif
1533
1534 /*
1535  * Commands handling.
1536  */
1537
1538 static __inline void
1539 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1540 {
1541         /* Command field: DW0 [7:0] */
1542         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1543         cmd->cmd_dword[0] |= htole64(cmd_type);
1544 }
1545
1546 static __inline void
1547 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1548 {
1549         /* Device ID field: DW0 [63:32] */
1550         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1551         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1552 }
1553
1554 static __inline void
1555 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1556 {
1557         /* Size field: DW1 [4:0] */
1558         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1559         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1560 }
1561
1562 static __inline void
1563 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1564 {
1565         /* ID field: DW1 [31:0] */
1566         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1567         cmd->cmd_dword[1] |= htole64(id);
1568 }
1569
1570 static __inline void
1571 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1572 {
1573         /* Physical ID field: DW1 [63:32] */
1574         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1575         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1576 }
1577
1578 static __inline void
1579 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1580 {
1581         /* Collection field: DW2 [16:0] */
1582         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1583         cmd->cmd_dword[2] |= htole64(col_id);
1584 }
1585
1586 static __inline void
1587 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1588 {
1589         /* Target Address field: DW2 [47:16] */
1590         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1591         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1592 }
1593
1594 static __inline void
1595 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1596 {
1597         /* ITT Address field: DW2 [47:8] */
1598         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1599         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1600 }
1601
1602 static __inline void
1603 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1604 {
1605         /* Valid field: DW2 [63] */
1606         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1607         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1608 }
1609
1610 static inline bool
1611 its_cmd_queue_full(struct gicv3_its_softc *sc)
1612 {
1613         size_t read_idx, next_write_idx;
1614
1615         /* Get the index of the next command */
1616         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1617             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1618         /* And the index of the current command being read */
1619         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1620
1621         /*
1622          * The queue is full when the write offset points
1623          * at the command before the current read offset.
1624          */
1625         return (next_write_idx == read_idx);
1626 }
1627
1628 static inline void
1629 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1630 {
1631
1632         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1633                 /* Clean D-cache under command. */
1634                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1635         } else {
1636                 /* DSB inner shareable, store */
1637                 dsb(ishst);
1638         }
1639
1640 }
1641
1642 static inline uint64_t
1643 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1644 {
1645         uint64_t off;
1646
1647         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1648
1649         return (off);
1650 }
1651
1652 static void
1653 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1654     struct its_cmd *cmd_last)
1655 {
1656         struct gicv3_its_softc *sc;
1657         uint64_t first, last, read;
1658         size_t us_left;
1659
1660         sc = device_get_softc(dev);
1661
1662         /*
1663          * XXX ARM64TODO: This is obviously a significant delay.
1664          * The reason for that is that currently the time frames for
1665          * the command to complete are not known.
1666          */
1667         us_left = 1000000;
1668
1669         first = its_cmd_cwriter_offset(sc, cmd_first);
1670         last = its_cmd_cwriter_offset(sc, cmd_last);
1671
1672         for (;;) {
1673                 read = gic_its_read_8(sc, GITS_CREADR);
1674                 if (first < last) {
1675                         if (read < first || read >= last)
1676                                 break;
1677                 } else if (read < first && read >= last)
1678                         break;
1679
1680                 if (us_left-- == 0) {
1681                         /* This means timeout */
1682                         device_printf(dev,
1683                             "Timeout while waiting for CMD completion.\n");
1684                         return;
1685                 }
1686                 DELAY(1);
1687         }
1688 }
1689
1690 static struct its_cmd *
1691 its_cmd_alloc_locked(device_t dev)
1692 {
1693         struct gicv3_its_softc *sc;
1694         struct its_cmd *cmd;
1695         size_t us_left;
1696
1697         sc = device_get_softc(dev);
1698
1699         /*
1700          * XXX ARM64TODO: This is obviously a significant delay.
1701          * The reason for that is that currently the time frames for
1702          * the command to complete (and therefore free the descriptor)
1703          * are not known.
1704          */
1705         us_left = 1000000;
1706
1707         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1708         while (its_cmd_queue_full(sc)) {
1709                 if (us_left-- == 0) {
1710                         /* Timeout while waiting for free command */
1711                         device_printf(dev,
1712                             "Timeout while waiting for free command\n");
1713                         return (NULL);
1714                 }
1715                 DELAY(1);
1716         }
1717
1718         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1719         sc->sc_its_cmd_next_idx++;
1720         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1721
1722         return (cmd);
1723 }
1724
1725 static uint64_t
1726 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1727 {
1728         uint64_t target;
1729         uint8_t cmd_type;
1730         u_int size;
1731
1732         cmd_type = desc->cmd_type;
1733         target = ITS_TARGET_NONE;
1734
1735         switch (cmd_type) {
1736         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1737                 target = desc->cmd_desc_movi.col->col_target;
1738                 cmd_format_command(cmd, ITS_CMD_MOVI);
1739                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1740                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1741                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1742                 break;
1743         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1744                 target = desc->cmd_desc_sync.col->col_target;
1745                 cmd_format_command(cmd, ITS_CMD_SYNC);
1746                 cmd_format_target(cmd, target);
1747                 break;
1748         case ITS_CMD_MAPD:      /* Assign ITT to device */
1749                 cmd_format_command(cmd, ITS_CMD_MAPD);
1750                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1751                 /*
1752                  * Size describes number of bits to encode interrupt IDs
1753                  * supported by the device minus one.
1754                  * When V (valid) bit is zero, this field should be written
1755                  * as zero.
1756                  */
1757                 if (desc->cmd_desc_mapd.valid != 0) {
1758                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1759                         size = MAX(1, size) - 1;
1760                 } else
1761                         size = 0;
1762
1763                 cmd_format_size(cmd, size);
1764                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1765                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1766                 break;
1767         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1768                 target = desc->cmd_desc_mapc.col->col_target;
1769                 cmd_format_command(cmd, ITS_CMD_MAPC);
1770                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1771                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1772                 cmd_format_target(cmd, target);
1773                 break;
1774         case ITS_CMD_MAPTI:
1775                 target = desc->cmd_desc_mapvi.col->col_target;
1776                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1777                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1778                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1779                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1780                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1781                 break;
1782         case ITS_CMD_MAPI:
1783                 target = desc->cmd_desc_mapi.col->col_target;
1784                 cmd_format_command(cmd, ITS_CMD_MAPI);
1785                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1786                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1787                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1788                 break;
1789         case ITS_CMD_INV:
1790                 target = desc->cmd_desc_inv.col->col_target;
1791                 cmd_format_command(cmd, ITS_CMD_INV);
1792                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1793                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1794                 break;
1795         case ITS_CMD_INVALL:
1796                 cmd_format_command(cmd, ITS_CMD_INVALL);
1797                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1798                 break;
1799         default:
1800                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1801         }
1802
1803         return (target);
1804 }
1805
1806 static int
1807 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1808 {
1809         struct gicv3_its_softc *sc;
1810         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1811         struct its_col col_sync;
1812         struct its_cmd_desc desc_sync;
1813         uint64_t target, cwriter;
1814
1815         sc = device_get_softc(dev);
1816         mtx_lock_spin(&sc->sc_its_cmd_lock);
1817         cmd = its_cmd_alloc_locked(dev);
1818         if (cmd == NULL) {
1819                 device_printf(dev, "could not allocate ITS command\n");
1820                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1821                 return (EBUSY);
1822         }
1823
1824         target = its_cmd_prepare(cmd, desc);
1825         its_cmd_sync(sc, cmd);
1826
1827         if (target != ITS_TARGET_NONE) {
1828                 cmd_sync = its_cmd_alloc_locked(dev);
1829                 if (cmd_sync != NULL) {
1830                         desc_sync.cmd_type = ITS_CMD_SYNC;
1831                         col_sync.col_target = target;
1832                         desc_sync.cmd_desc_sync.col = &col_sync;
1833                         its_cmd_prepare(cmd_sync, &desc_sync);
1834                         its_cmd_sync(sc, cmd_sync);
1835                 }
1836         }
1837
1838         /* Update GITS_CWRITER */
1839         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
1840         gic_its_write_8(sc, GITS_CWRITER, cwriter);
1841         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1842         mtx_unlock_spin(&sc->sc_its_cmd_lock);
1843
1844         its_cmd_wait_completion(dev, cmd, cmd_write);
1845
1846         return (0);
1847 }
1848
1849 /* Handlers to send commands */
1850 static void
1851 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
1852 {
1853         struct gicv3_its_softc *sc;
1854         struct its_cmd_desc desc;
1855         struct its_col *col;
1856
1857         sc = device_get_softc(dev);
1858         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1859
1860         desc.cmd_type = ITS_CMD_MOVI;
1861         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
1862         desc.cmd_desc_movi.col = col;
1863         desc.cmd_desc_movi.id = girq->gi_id;
1864
1865         its_cmd_send(dev, &desc);
1866 }
1867
1868 static void
1869 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
1870 {
1871         struct its_cmd_desc desc;
1872
1873         desc.cmd_type = ITS_CMD_MAPC;
1874         desc.cmd_desc_mapc.col = col;
1875         /*
1876          * Valid bit set - map the collection.
1877          * Valid bit cleared - unmap the collection.
1878          */
1879         desc.cmd_desc_mapc.valid = valid;
1880
1881         its_cmd_send(dev, &desc);
1882 }
1883
1884 static void
1885 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
1886 {
1887         struct gicv3_its_softc *sc;
1888         struct its_cmd_desc desc;
1889         struct its_col *col;
1890         u_int col_id;
1891
1892         sc = device_get_softc(dev);
1893
1894         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
1895         col = sc->sc_its_cols[col_id];
1896
1897         desc.cmd_type = ITS_CMD_MAPTI;
1898         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
1899         desc.cmd_desc_mapvi.col = col;
1900         /* The EventID sent to the device */
1901         desc.cmd_desc_mapvi.id = girq->gi_id;
1902         /* The physical interrupt presented to softeware */
1903         desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
1904
1905         its_cmd_send(dev, &desc);
1906 }
1907
1908 static void
1909 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
1910 {
1911         struct its_cmd_desc desc;
1912
1913         desc.cmd_type = ITS_CMD_MAPD;
1914         desc.cmd_desc_mapd.its_dev = its_dev;
1915         desc.cmd_desc_mapd.valid = valid;
1916
1917         its_cmd_send(dev, &desc);
1918 }
1919
1920 static void
1921 its_cmd_inv(device_t dev, struct its_dev *its_dev,
1922     struct gicv3_its_irqsrc *girq)
1923 {
1924         struct gicv3_its_softc *sc;
1925         struct its_cmd_desc desc;
1926         struct its_col *col;
1927
1928         sc = device_get_softc(dev);
1929         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1930
1931         desc.cmd_type = ITS_CMD_INV;
1932         /* The EventID sent to the device */
1933         desc.cmd_desc_inv.pid = girq->gi_id;
1934         desc.cmd_desc_inv.its_dev = its_dev;
1935         desc.cmd_desc_inv.col = col;
1936
1937         its_cmd_send(dev, &desc);
1938 }
1939
1940 static void
1941 its_cmd_invall(device_t dev, struct its_col *col)
1942 {
1943         struct its_cmd_desc desc;
1944
1945         desc.cmd_type = ITS_CMD_INVALL;
1946         desc.cmd_desc_invall.col = col;
1947
1948         its_cmd_send(dev, &desc);
1949 }
1950
1951 #ifdef FDT
1952 static device_probe_t gicv3_its_fdt_probe;
1953 static device_attach_t gicv3_its_fdt_attach;
1954
1955 static device_method_t gicv3_its_fdt_methods[] = {
1956         /* Device interface */
1957         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
1958         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
1959
1960         /* End */
1961         DEVMETHOD_END
1962 };
1963
1964 #define its_baseclasses its_fdt_baseclasses
1965 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
1966     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1967 #undef its_baseclasses
1968 static devclass_t gicv3_its_fdt_devclass;
1969
1970 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver,
1971     gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1972
1973 static int
1974 gicv3_its_fdt_probe(device_t dev)
1975 {
1976
1977         if (!ofw_bus_status_okay(dev))
1978                 return (ENXIO);
1979
1980         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
1981                 return (ENXIO);
1982
1983         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
1984         return (BUS_PROBE_DEFAULT);
1985 }
1986
1987 static int
1988 gicv3_its_fdt_attach(device_t dev)
1989 {
1990         struct gicv3_its_softc *sc;
1991         phandle_t xref;
1992         int err;
1993
1994         sc = device_get_softc(dev);
1995         sc->dev = dev;
1996         err = gicv3_its_attach(dev);
1997         if (err != 0)
1998                 return (err);
1999
2000         /* Register this device as a interrupt controller */
2001         xref = OF_xref_from_node(ofw_bus_get_node(dev));
2002         sc->sc_pic = intr_pic_register(dev, xref);
2003         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2004             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2005         if (err != 0) {
2006                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2007                 return (err);
2008         }
2009
2010         /* Register this device to handle MSI interrupts */
2011         err = intr_msi_register(dev, xref);
2012         if (err != 0) {
2013                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2014                 return (err);
2015         }
2016
2017         return (0);
2018 }
2019 #endif
2020
2021 #ifdef DEV_ACPI
2022 static device_probe_t gicv3_its_acpi_probe;
2023 static device_attach_t gicv3_its_acpi_attach;
2024
2025 static device_method_t gicv3_its_acpi_methods[] = {
2026         /* Device interface */
2027         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
2028         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
2029
2030         /* End */
2031         DEVMETHOD_END
2032 };
2033
2034 #define its_baseclasses its_acpi_baseclasses
2035 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2036     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2037 #undef its_baseclasses
2038 static devclass_t gicv3_its_acpi_devclass;
2039
2040 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver,
2041     gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2042
2043 static int
2044 gicv3_its_acpi_probe(device_t dev)
2045 {
2046
2047         if (gic_get_bus(dev) != GIC_BUS_ACPI)
2048                 return (EINVAL);
2049
2050         if (gic_get_hw_rev(dev) < 3)
2051                 return (EINVAL);
2052
2053         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2054         return (BUS_PROBE_DEFAULT);
2055 }
2056
2057 static int
2058 gicv3_its_acpi_attach(device_t dev)
2059 {
2060         struct gicv3_its_softc *sc;
2061         struct gic_v3_devinfo *di;
2062         int err;
2063
2064         sc = device_get_softc(dev);
2065         sc->dev = dev;
2066         err = gicv3_its_attach(dev);
2067         if (err != 0)
2068                 return (err);
2069
2070         di = device_get_ivars(dev);
2071         sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2072         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2073             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2074         if (err != 0) {
2075                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2076                 return (err);
2077         }
2078
2079         /* Register this device to handle MSI interrupts */
2080         err = intr_msi_register(dev, di->msi_xref);
2081         if (err != 0) {
2082                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2083                 return (err);
2084         }
2085
2086         return (0);
2087 }
2088 #endif