sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  *
   4  * This software was developed by Andrew Turner under
   5  * the sponsorship of the FreeBSD Foundation.
   6  *
   7  * This software was developed by Semihalf under
   8  * the sponsorship of the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include "opt_acpi.h"
  33 #include "opt_platform.h"
  34 #include "opt_iommu.h"
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/bus.h>
  42 #include <sys/cpuset.h>
  43 #include <sys/domainset.h>
  44 #include <sys/endian.h>
  45 #include <sys/kernel.h>
  46 #include <sys/lock.h>
  47 #include <sys/malloc.h>
  48 #include <sys/module.h>
  49 #include <sys/mutex.h>
  50 #include <sys/proc.h>
  51 #include <sys/taskqueue.h>
  52 #include <sys/tree.h>
  53 #include <sys/queue.h>
  54 #include <sys/rman.h>
  55 #include <sys/sbuf.h>
  56 #include <sys/smp.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/vmem.h>
  59
  60 #include <vm/vm.h>
  61 #include <vm/pmap.h>
  62 #include <vm/vm_page.h>
  63
  64 #include <machine/bus.h>
  65 #include <machine/intr.h>
  66
  67 #include <arm/arm/gic_common.h>
  68 #include <arm64/arm64/gic_v3_reg.h>
  69 #include <arm64/arm64/gic_v3_var.h>
  70
  71 #ifdef FDT
  72 #include <dev/ofw/openfirm.h>
  73 #include <dev/ofw/ofw_bus.h>
  74 #include <dev/ofw/ofw_bus_subr.h>
  75 #endif
  76 #include <dev/pci/pcireg.h>
  77 #include <dev/pci/pcivar.h>
  78
  79 #ifdef IOMMU
  80 #include <dev/iommu/iommu.h>
  81 #include <dev/iommu/iommu_gas.h>
  82 #endif
  83
  84 #include "pcib_if.h"
  85 #include "pic_if.h"
  86 #include "msi_if.h"
  87
  88 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  89     "ARM GICv3 Interrupt Translation Service");
  90
  91 #define LPI_NIRQS               (64 * 1024)
  92
  93 /* The size and alignment of the command circular buffer */
  94 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  95 #define ITS_CMDQ_ALIGN          (64 * 1024)
  96
  97 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  98 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  99 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 100
 101 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
 102 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
 103 #define LPI_PENDTAB_ALIGN       (64 * 1024)
 104 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 105
 106 #define LPI_INT_TRANS_TAB_ALIGN 256
 107 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
 108
 109 /* ITS commands encoding */
 110 #define ITS_CMD_MOVI            (0x01)
 111 #define ITS_CMD_SYNC            (0x05)
 112 #define ITS_CMD_MAPD            (0x08)
 113 #define ITS_CMD_MAPC            (0x09)
 114 #define ITS_CMD_MAPTI           (0x0a)
 115 #define ITS_CMD_MAPI            (0x0b)
 116 #define ITS_CMD_INV             (0x0c)
 117 #define ITS_CMD_INVALL          (0x0d)
 118 /* Command */
 119 #define CMD_COMMAND_MASK        (0xFFUL)
 120 /* PCI device ID */
 121 #define CMD_DEVID_SHIFT         (32)
 122 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 123 /* Size of IRQ ID bitfield */
 124 #define CMD_SIZE_MASK           (0xFFUL)
 125 /* Virtual LPI ID */
 126 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 127 /* Physical LPI ID */
 128 #define CMD_PID_SHIFT           (32)
 129 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 130 /* Collection */
 131 #define CMD_COL_MASK            (0xFFFFUL)
 132 /* Target (CPU or Re-Distributor) */
 133 #define CMD_TARGET_SHIFT        (16)
 134 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 135 /* Interrupt Translation Table address */
 136 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 137 /* Valid command bit */
 138 #define CMD_VALID_SHIFT         (63)
 139 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 140
 141 #define ITS_TARGET_NONE         0xFBADBEEF
 142
 143 /* LPI chunk owned by ITS device */
 144 struct lpi_chunk {
 145         u_int   lpi_base;
 146         u_int   lpi_free;       /* First free LPI in set */
 147         u_int   lpi_num;        /* Total number of LPIs in chunk */
 148         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 149 };
 150
 151 /* ITS device */
 152 struct its_dev {
 153         TAILQ_ENTRY(its_dev)    entry;
 154         /* PCI device */
 155         device_t                pci_dev;
 156         /* Device ID (i.e. PCI device ID) */
 157         uint32_t                devid;
 158         /* List of assigned LPIs */
 159         struct lpi_chunk        lpis;
 160         /* Virtual address of ITT */
 161         vm_offset_t             itt;
 162         size_t                  itt_size;
 163 };
 164
 165 /*
 166  * ITS command descriptor.
 167  * Idea for command description passing taken from Linux.
 168  */
 169 struct its_cmd_desc {
 170         uint8_t cmd_type;
 171
 172         union {
 173                 struct {
 174                         struct its_dev *its_dev;
 175                         struct its_col *col;
 176                         uint32_t id;
 177                 } cmd_desc_movi;
 178
 179                 struct {
 180                         struct its_col *col;
 181                 } cmd_desc_sync;
 182
 183                 struct {
 184                         struct its_col *col;
 185                         uint8_t valid;
 186                 } cmd_desc_mapc;
 187
 188                 struct {
 189                         struct its_dev *its_dev;
 190                         struct its_col *col;
 191                         uint32_t pid;
 192                         uint32_t id;
 193                 } cmd_desc_mapvi;
 194
 195                 struct {
 196                         struct its_dev *its_dev;
 197                         struct its_col *col;
 198                         uint32_t pid;
 199                 } cmd_desc_mapi;
 200
 201                 struct {
 202                         struct its_dev *its_dev;
 203                         uint8_t valid;
 204                 } cmd_desc_mapd;
 205
 206                 struct {
 207                         struct its_dev *its_dev;
 208                         struct its_col *col;
 209                         uint32_t pid;
 210                 } cmd_desc_inv;
 211
 212                 struct {
 213                         struct its_col *col;
 214                 } cmd_desc_invall;
 215         };
 216 };
 217
 218 /* ITS command. Each command is 32 bytes long */
 219 struct its_cmd {
 220         uint64_t        cmd_dword[4];   /* ITS command double word */
 221 };
 222
 223 /* An ITS private table */
 224 struct its_ptable {
 225         vm_offset_t     ptab_vaddr;
 226         unsigned long   ptab_size;
 227 };
 228
 229 /* ITS collection description. */
 230 struct its_col {
 231         uint64_t        col_target;     /* Target Re-Distributor */
 232         uint64_t        col_id;         /* Collection ID */
 233 };
 234
 235 struct gicv3_its_irqsrc {
 236         struct intr_irqsrc      gi_isrc;
 237         u_int                   gi_id;
 238         u_int                   gi_lpi;
 239         struct its_dev          *gi_its_dev;
 240         TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
 241 };
 242
 243 struct gicv3_its_softc {
 244         device_t        dev;
 245         struct intr_pic *sc_pic;
 246         struct resource *sc_its_res;
 247
 248         cpuset_t        sc_cpus;
 249         struct domainset *sc_ds;
 250         u_int           gic_irq_cpu;
 251
 252         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 253         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 254
 255         /*
 256          * TODO: We should get these from the parent as we only want a
 257          * single copy of each across the interrupt controller.
 258          */
 259         uint8_t         *sc_conf_base;
 260         vm_offset_t sc_pend_base[MAXCPU];
 261
 262         /* Command handling */
 263         struct mtx sc_its_cmd_lock;
 264         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 265         size_t sc_its_cmd_next_idx;
 266
 267         vmem_t *sc_irq_alloc;
 268         struct gicv3_its_irqsrc **sc_irqs;
 269         u_int   sc_irq_base;
 270         u_int   sc_irq_length;
 271         u_int   sc_irq_count;
 272
 273         struct mtx sc_its_dev_lock;
 274         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 275         TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
 276
 277 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 278 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 279 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 280         u_int sc_its_flags;
 281         bool    trace_enable;
 282         vm_page_t ma; /* fake msi page */
 283 };
 284
 285 static void *conf_base;
 286
 287 typedef void (its_quirk_func_t)(device_t);
 288 static its_quirk_func_t its_quirk_cavium_22375;
 289
 290 static const struct {
 291         const char *desc;
 292         uint32_t iidr;
 293         uint32_t iidr_mask;
 294         its_quirk_func_t *func;
 295 } its_quirks[] = {
 296         {
 297                 /* Cavium ThunderX Pass 1.x */
 298                 .desc = "Cavium ThunderX errata: 22375, 24313",
 299                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 300                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 301                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 302                 .func = its_quirk_cavium_22375,
 303         },
 304 };
 305
 306 #define gic_its_read_4(sc, reg)                 \
 307     bus_read_4((sc)->sc_its_res, (reg))
 308 #define gic_its_read_8(sc, reg)                 \
 309     bus_read_8((sc)->sc_its_res, (reg))
 310
 311 #define gic_its_write_4(sc, reg, val)           \
 312     bus_write_4((sc)->sc_its_res, (reg), (val))
 313 #define gic_its_write_8(sc, reg, val)           \
 314     bus_write_8((sc)->sc_its_res, (reg), (val))
 315
 316 static device_attach_t gicv3_its_attach;
 317 static device_detach_t gicv3_its_detach;
 318
 319 static pic_disable_intr_t gicv3_its_disable_intr;
 320 static pic_enable_intr_t gicv3_its_enable_intr;
 321 static pic_map_intr_t gicv3_its_map_intr;
 322 static pic_setup_intr_t gicv3_its_setup_intr;
 323 static pic_post_filter_t gicv3_its_post_filter;
 324 static pic_post_ithread_t gicv3_its_post_ithread;
 325 static pic_pre_ithread_t gicv3_its_pre_ithread;
 326 static pic_bind_intr_t gicv3_its_bind_intr;
 327 #ifdef SMP
 328 static pic_init_secondary_t gicv3_its_init_secondary;
 329 #endif
 330 static msi_alloc_msi_t gicv3_its_alloc_msi;
 331 static msi_release_msi_t gicv3_its_release_msi;
 332 static msi_alloc_msix_t gicv3_its_alloc_msix;
 333 static msi_release_msix_t gicv3_its_release_msix;
 334 static msi_map_msi_t gicv3_its_map_msi;
 335 #ifdef IOMMU
 336 static msi_iommu_init_t gicv3_iommu_init;
 337 static msi_iommu_deinit_t gicv3_iommu_deinit;
 338 #endif
 339
 340 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 341 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 342 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 343 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 344 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 345 static void its_cmd_invall(device_t, struct its_col *);
 346
 347 static device_method_t gicv3_its_methods[] = {
 348         /* Device interface */
 349         DEVMETHOD(device_detach,        gicv3_its_detach),
 350
 351         /* Interrupt controller interface */
 352         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 353         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 354         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 355         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 356         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 357         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 358         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 359 #ifdef SMP
 360         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 361         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 362 #endif
 363
 364         /* MSI/MSI-X */
 365         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 366         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 367         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 368         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 369         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 370 #ifdef IOMMU
 371         DEVMETHOD(msi_iommu_init,       gicv3_iommu_init),
 372         DEVMETHOD(msi_iommu_deinit,     gicv3_iommu_deinit),
 373 #endif
 374
 375         /* End */
 376         DEVMETHOD_END
 377 };
 378
 379 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 380     sizeof(struct gicv3_its_softc));
 381
 382 static void
 383 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 384 {
 385         vm_paddr_t cmd_paddr;
 386         uint64_t reg, tmp;
 387
 388         /* Set up the command circular buffer */
 389         sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
 390             sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
 391             0);
 392         sc->sc_its_cmd_next_idx = 0;
 393
 394         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 395
 396         /* Set the base of the command buffer */
 397         reg = GITS_CBASER_VALID |
 398             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 399             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 400             (ITS_CMDQ_SIZE / 4096 - 1);
 401         gic_its_write_8(sc, GITS_CBASER, reg);
 402
 403         /* Read back to check for fixed value fields */
 404         tmp = gic_its_read_8(sc, GITS_CBASER);
 405
 406         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 407             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 408                 /* Check if the hardware reported non-shareable */
 409                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 410                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 411                         /* If so remove the cache attribute */
 412                         reg &= ~GITS_CBASER_CACHE_MASK;
 413                         reg &= ~GITS_CBASER_SHARE_MASK;
 414                         /* Set to Non-cacheable, Non-shareable */
 415                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 416                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 417
 418                         gic_its_write_8(sc, GITS_CBASER, reg);
 419                 }
 420
 421                 /* The command queue has to be flushed after each command */
 422                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 423         }
 424
 425         /* Get the next command from the start of the buffer */
 426         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 427 }
 428
 429 static int
 430 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 431 {
 432         vm_offset_t table;
 433         vm_paddr_t paddr;
 434         uint64_t cache, reg, share, tmp, type;
 435         size_t esize, its_tbl_size, nidents, nitspages, npages;
 436         int i, page_size;
 437         int devbits;
 438
 439         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 440                 /*
 441                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 442                  * are to be 21 bits in length. The entry size of the ITS
 443                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 444                  * is supposed to be 8 bytes in length (for device table).
 445                  * Finally the page size that is to be used by ITS to access
 446                  * this table will be set to 64KB.
 447                  *
 448                  * This gives 0x200000 entries of size 0x8 bytes covered by
 449                  * 256 pages each of which 64KB in size. The number of pages
 450                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 451                  * that case this value would be 0xFF but on ThunderX the
 452                  * maximum value that HW accepts is 0xFD.
 453                  *
 454                  * Set an arbitrary number of device ID bits to 20 in order
 455                  * to limit the number of entries in ITS device table to
 456                  * 0x100000 and the table size to 8MB.
 457                  */
 458                 devbits = 20;
 459                 cache = 0;
 460         } else {
 461                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 462                 cache = GITS_BASER_CACHE_WAWB;
 463         }
 464         share = GITS_BASER_SHARE_IS;
 465         page_size = PAGE_SIZE_64K;
 466
 467         for (i = 0; i < GITS_BASER_NUM; i++) {
 468                 reg = gic_its_read_8(sc, GITS_BASER(i));
 469                 /* The type of table */
 470                 type = GITS_BASER_TYPE(reg);
 471                 /* The table entry size */
 472                 esize = GITS_BASER_ESIZE(reg);
 473
 474                 switch(type) {
 475                 case GITS_BASER_TYPE_DEV:
 476                         nidents = (1 << devbits);
 477                         its_tbl_size = esize * nidents;
 478                         its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K);
 479                         break;
 480                 case GITS_BASER_TYPE_VP:
 481                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 482                 case GITS_BASER_TYPE_IC:
 483                         its_tbl_size = page_size;
 484                         break;
 485                 default:
 486                         continue;
 487                 }
 488                 npages = howmany(its_tbl_size, PAGE_SIZE);
 489
 490                 /* Allocate the table */
 491                 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
 492                     M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
 493                     (1ul << 48) - 1, PAGE_SIZE_64K, 0);
 494
 495                 sc->sc_its_ptab[i].ptab_vaddr = table;
 496                 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;
 497
 498                 paddr = vtophys(table);
 499
 500                 while (1) {
 501                         nitspages = howmany(its_tbl_size, page_size);
 502
 503                         /* Clear the fields we will be setting */
 504                         reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
 505                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 506                             GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK |
 507                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 508                             GITS_BASER_SIZE_MASK);
 509                         /* Set the new values */
 510                         reg |= GITS_BASER_VALID |
 511                             (cache << GITS_BASER_CACHE_SHIFT) |
 512                             (type << GITS_BASER_TYPE_SHIFT) |
 513                             ((esize - 1) << GITS_BASER_ESIZE_SHIFT) |
 514                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 515                             (nitspages - 1);
 516
 517                         switch (page_size) {
 518                         case PAGE_SIZE_4K:      /* 4KB */
 519                                 reg |=
 520                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 521                                 break;
 522                         case PAGE_SIZE_16K:     /* 16KB */
 523                                 reg |=
 524                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 525                                 break;
 526                         case PAGE_SIZE_64K:     /* 64KB */
 527                                 reg |=
 528                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 529                                 break;
 530                         }
 531
 532                         gic_its_write_8(sc, GITS_BASER(i), reg);
 533
 534                         /* Read back to check */
 535                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 536
 537                         /* Do the shareability masks line up? */
 538                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 539                             (reg & GITS_BASER_SHARE_MASK)) {
 540                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 541                                     GITS_BASER_SHARE_SHIFT;
 542                                 continue;
 543                         }
 544
 545                         if ((tmp & GITS_BASER_PSZ_MASK) !=
 546                             (reg & GITS_BASER_PSZ_MASK)) {
 547                                 switch (page_size) {
 548                                 case PAGE_SIZE_16K:
 549                                         page_size = PAGE_SIZE_4K;
 550                                         continue;
 551                                 case PAGE_SIZE_64K:
 552                                         page_size = PAGE_SIZE_16K;
 553                                         continue;
 554                                 }
 555                         }
 556
 557                         if (tmp != reg) {
 558                                 device_printf(dev, "GITS_BASER%d: "
 559                                     "unable to be updated: %lx != %lx\n",
 560                                     i, reg, tmp);
 561                                 return (ENXIO);
 562                         }
 563
 564                         /* We should have made all needed changes */
 565                         break;
 566                 }
 567         }
 568
 569         return (0);
 570 }
 571
 572 static void
 573 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 574 {
 575         void *conf_table;
 576
 577         conf_table = atomic_load_ptr(&conf_base);
 578         if (conf_table == NULL) {
 579                 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
 580                     M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
 581                     LPI_CONFTAB_ALIGN, 0);
 582
 583                 if (atomic_cmpset_ptr((uintptr_t *)&conf_base,
 584                     (uintptr_t)NULL, (uintptr_t)conf_table) == 0) {
 585                         contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS);
 586                         conf_table = atomic_load_ptr(&conf_base);
 587                 }
 588         }
 589         sc->sc_conf_base = conf_table;
 590
 591         /* Set the default configuration */
 592         memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 593             LPI_CONFTAB_SIZE);
 594
 595         /* Flush the table to memory */
 596         cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
 597 }
 598
 599 static void
 600 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 601 {
 602         int i;
 603
 604         for (i = 0; i <= mp_maxid; i++) {
 605                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 606                         continue;
 607
 608                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 609                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 610                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 611
 612                 /* Flush so the ITS can see the memory */
 613                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
 614                     LPI_PENDTAB_SIZE);
 615         }
 616 }
 617
 618 static void
 619 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
 620 {
 621         device_t gicv3;
 622         uint64_t xbaser, tmp;
 623         uint32_t ctlr;
 624         u_int cpuid;
 625
 626         gicv3 = device_get_parent(dev);
 627         cpuid = PCPU_GET(cpuid);
 628
 629         /* Disable LPIs */
 630         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 631         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 632         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 633
 634         /* Make sure changes are observable my the GIC */
 635         dsb(sy);
 636
 637         /*
 638          * Set the redistributor base
 639          */
 640         xbaser = vtophys(sc->sc_conf_base) |
 641             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 642             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 643             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 644         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 645
 646         /* Check the cache attributes we set */
 647         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 648
 649         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 650             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 651                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 652                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 653                         /* We need to mark as non-cacheable */
 654                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 655                             GICR_PROPBASER_CACHE_MASK);
 656                         /* Non-cacheable */
 657                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 658                             GICR_PROPBASER_CACHE_SHIFT;
 659                         /* Non-shareable */
 660                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 661                             GICR_PROPBASER_SHARE_SHIFT;
 662                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 663                 }
 664                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 665         }
 666
 667         /*
 668          * Set the LPI pending table base
 669          */
 670         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 671             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 672             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 673
 674         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 675
 676         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 677
 678         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 679             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 680                 /* Clear the cahce and shareability bits */
 681                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 682                     GICR_PENDBASER_SHARE_MASK);
 683                 /* Mark as non-shareable */
 684                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 685                 /* And non-cacheable */
 686                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 687                     GICR_PENDBASER_CACHE_SHIFT;
 688         }
 689
 690         /* Enable LPIs */
 691         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 692         ctlr |= GICR_CTLR_LPI_ENABLE;
 693         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 694
 695         /* Make sure the GIC has seen everything */
 696         dsb(sy);
 697 }
 698
 699 static int
 700 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 701 {
 702         device_t gicv3;
 703         vm_paddr_t target;
 704         u_int cpuid;
 705         struct redist_pcpu *rpcpu;
 706
 707         gicv3 = device_get_parent(dev);
 708         cpuid = PCPU_GET(cpuid);
 709         if (!CPU_ISSET(cpuid, &sc->sc_cpus))
 710                 return (0);
 711
 712         /* Check if the ITS is enabled on this CPU */
 713         if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
 714                 return (ENXIO);
 715
 716         rpcpu = gicv3_get_redist(dev);
 717
 718         /* Do per-cpu LPI init once */
 719         if (!rpcpu->lpi_enabled) {
 720                 its_init_cpu_lpi(dev, sc);
 721                 rpcpu->lpi_enabled = true;
 722         }
 723
 724         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 725                 /* This ITS wants the redistributor physical address */
 726                 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
 727                     rpcpu->offset);
 728         } else {
 729                 /* This ITS wants the unique processor number */
 730                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
 731                     CMD_TARGET_SHIFT;
 732         }
 733
 734         sc->sc_its_cols[cpuid]->col_target = target;
 735         sc->sc_its_cols[cpuid]->col_id = cpuid;
 736
 737         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 738         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 739
 740         return (0);
 741 }
 742
 743 static int
 744 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
 745 {
 746         struct gicv3_its_softc *sc;
 747         int rv;
 748
 749         sc = arg1;
 750
 751         rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
 752         if (rv != 0 || req->newptr == NULL)
 753                 return (rv);
 754         if (sc->trace_enable)
 755                 gic_its_write_8(sc, GITS_TRKCTLR, 3);
 756         else
 757                 gic_its_write_8(sc, GITS_TRKCTLR, 0);
 758
 759         return (0);
 760 }
 761
 762 static int
 763 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
 764 {
 765         struct gicv3_its_softc *sc;
 766         struct sbuf *sb;
 767         int err;
 768
 769         sc = arg1;
 770         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 771         if (sb == NULL) {
 772                 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
 773                 return (ENOMEM);
 774         }
 775         sbuf_cat(sb, "\n");
 776         sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
 777             gic_its_read_4(sc, GITS_TRKCTLR));
 778         sbuf_printf(sb, "GITS_TRKR:    0x%08X\n",
 779             gic_its_read_4(sc, GITS_TRKR));
 780         sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
 781             gic_its_read_4(sc, GITS_TRKDIDR));
 782         sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
 783             gic_its_read_4(sc, GITS_TRKPIDR));
 784         sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
 785             gic_its_read_4(sc, GITS_TRKVIDR));
 786         sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
 787            gic_its_read_4(sc, GITS_TRKTGTR));
 788
 789         err = sbuf_finish(sb);
 790         if (err)
 791                 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
 792         sbuf_delete(sb);
 793         return(err);
 794 }
 795
 796 static int
 797 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
 798 {
 799         struct sysctl_oid *oid, *child;
 800         struct sysctl_ctx_list *ctx_list;
 801
 802         ctx_list = device_get_sysctl_ctx(sc->dev);
 803         child = device_get_sysctl_tree(sc->dev);
 804         oid = SYSCTL_ADD_NODE(ctx_list,
 805             SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
 806             CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
 807         if (oid == NULL)
 808                 return (ENXIO);
 809
 810         /* Add registers */
 811         SYSCTL_ADD_PROC(ctx_list,
 812             SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
 813             CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 814             gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
 815         SYSCTL_ADD_PROC(ctx_list,
 816             SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
 817             CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 818             gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
 819
 820         return (0);
 821 }
 822
 823 static int
 824 gicv3_its_attach(device_t dev)
 825 {
 826         struct gicv3_its_softc *sc;
 827         int domain, err, i, rid;
 828         uint64_t phys;
 829         uint32_t ctlr, iidr;
 830
 831         sc = device_get_softc(dev);
 832
 833         sc->sc_irq_length = gicv3_get_nirqs(dev);
 834         sc->sc_irq_base = GIC_FIRST_LPI;
 835         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
 836
 837         rid = 0;
 838         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 839             RF_ACTIVE);
 840         if (sc->sc_its_res == NULL) {
 841                 device_printf(dev, "Could not allocate memory\n");
 842                 return (ENXIO);
 843         }
 844
 845         phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
 846             GITS_TRANSLATER, PAGE_SIZE);
 847         sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
 848         vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
 849
 850         CPU_COPY(&all_cpus, &sc->sc_cpus);
 851         iidr = gic_its_read_4(sc, GITS_IIDR);
 852         for (i = 0; i < nitems(its_quirks); i++) {
 853                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 854                         if (bootverbose) {
 855                                 device_printf(dev, "Applying %s\n",
 856                                     its_quirks[i].desc);
 857                         }
 858                         its_quirks[i].func(dev);
 859                         break;
 860                 }
 861         }
 862
 863         if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
 864                 sc->sc_ds = DOMAINSET_PREF(domain);
 865         } else {
 866                 sc->sc_ds = DOMAINSET_RR();
 867         }
 868
 869         /*
 870          * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
 871          * coming in via, for instance, a kexec/kboot style setup where a
 872          * previous kernel has configured then relinquished control.  Clear it
 873          * so that we can reconfigure GITS_BASER*.
 874          */
 875         ctlr = gic_its_read_4(sc, GITS_CTLR);
 876         if ((ctlr & GITS_CTLR_EN) != 0) {
 877                 ctlr &= ~GITS_CTLR_EN;
 878                 gic_its_write_4(sc, GITS_CTLR, ctlr);
 879         }
 880
 881         /* Allocate the private tables */
 882         err = gicv3_its_table_init(dev, sc);
 883         if (err != 0)
 884                 return (err);
 885
 886         /* Protects access to the device list */
 887         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 888
 889         /* Protects access to the ITS command circular buffer. */
 890         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
 891
 892         /* Allocate the command circular buffer */
 893         gicv3_its_cmdq_init(sc);
 894
 895         /* Allocate the per-CPU collections */
 896         for (int cpu = 0; cpu <= mp_maxid; cpu++)
 897                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
 898                         sc->sc_its_cols[cpu] = malloc_domainset(
 899                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
 900                             DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
 901                             M_WAITOK | M_ZERO);
 902                 else
 903                         sc->sc_its_cols[cpu] = NULL;
 904
 905         /* Enable the ITS */
 906         gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
 907
 908         /* Create the LPI configuration table */
 909         gicv3_its_conftable_init(sc);
 910
 911         /* And the pending tebles */
 912         gicv3_its_pendtables_init(sc);
 913
 914         /* Enable LPIs on this CPU */
 915         its_init_cpu(dev, sc);
 916
 917         TAILQ_INIT(&sc->sc_its_dev_list);
 918         TAILQ_INIT(&sc->sc_free_irqs);
 919
 920         /*
 921          * Create the vmem object to allocate INTRNG IRQs from. We try to
 922          * use all IRQs not already used by the GICv3.
 923          * XXX: This assumes there are no other interrupt controllers in the
 924          * system.
 925          */
 926         sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
 927             gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
 928
 929         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
 930             M_GICV3_ITS, M_WAITOK | M_ZERO);
 931
 932         /* For GIC-500 install tracking sysctls. */
 933         if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
 934             GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
 935                 gicv3_its_init_sysctl(sc);
 936
 937         return (0);
 938 }
 939
 940 static int
 941 gicv3_its_detach(device_t dev)
 942 {
 943
 944         return (ENXIO);
 945 }
 946
 947 static void
 948 its_quirk_cavium_22375(device_t dev)
 949 {
 950         struct gicv3_its_softc *sc;
 951         int domain;
 952
 953         sc = device_get_softc(dev);
 954         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
 955
 956         /*
 957          * We need to limit which CPUs we send these interrupts to on
 958          * the original dual socket ThunderX as it is unable to
 959          * forward them between the two sockets.
 960          */
 961         if (bus_get_domain(dev, &domain) == 0) {
 962                 if (domain < MAXMEMDOM) {
 963                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
 964                 } else {
 965                         CPU_ZERO(&sc->sc_cpus);
 966                 }
 967         }
 968 }
 969
 970 static void
 971 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
 972 {
 973         struct gicv3_its_softc *sc;
 974         struct gicv3_its_irqsrc *girq;
 975         uint8_t *conf;
 976
 977         sc = device_get_softc(dev);
 978         girq = (struct gicv3_its_irqsrc *)isrc;
 979         conf = sc->sc_conf_base;
 980
 981         conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
 982
 983         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
 984                 /* Clean D-cache under command. */
 985                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
 986         } else {
 987                 /* DSB inner shareable, store */
 988                 dsb(ishst);
 989         }
 990
 991         its_cmd_inv(dev, girq->gi_its_dev, girq);
 992 }
 993
 994 static void
 995 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
 996 {
 997         struct gicv3_its_softc *sc;
 998         struct gicv3_its_irqsrc *girq;
 999         uint8_t *conf;
1000
1001         sc = device_get_softc(dev);
1002         girq = (struct gicv3_its_irqsrc *)isrc;
1003         conf = sc->sc_conf_base;
1004
1005         conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1006
1007         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1008                 /* Clean D-cache under command. */
1009                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1010         } else {
1011                 /* DSB inner shareable, store */
1012                 dsb(ishst);
1013         }
1014
1015         its_cmd_inv(dev, girq->gi_its_dev, girq);
1016 }
1017
1018 static int
1019 gicv3_its_intr(void *arg, uintptr_t irq)
1020 {
1021         struct gicv3_its_softc *sc = arg;
1022         struct gicv3_its_irqsrc *girq;
1023         struct trapframe *tf;
1024
1025         irq -= sc->sc_irq_base;
1026         girq = sc->sc_irqs[irq];
1027         if (girq == NULL)
1028                 panic("gicv3_its_intr: Invalid interrupt %ld",
1029                     irq + sc->sc_irq_base);
1030
1031         tf = curthread->td_intr_frame;
1032         intr_isrc_dispatch(&girq->gi_isrc, tf);
1033         return (FILTER_HANDLED);
1034 }
1035
1036 static void
1037 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1038 {
1039         struct gicv3_its_irqsrc *girq;
1040
1041         girq = (struct gicv3_its_irqsrc *)isrc;
1042         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1043 }
1044
1045 static void
1046 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1047 {
1048
1049 }
1050
1051 static void
1052 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1053 {
1054         struct gicv3_its_irqsrc *girq;
1055
1056         girq = (struct gicv3_its_irqsrc *)isrc;
1057         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1058 }
1059
1060 static int
1061 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1062 {
1063         struct gicv3_its_softc *sc;
1064
1065         sc = device_get_softc(dev);
1066         if (CPU_EMPTY(&isrc->isrc_cpu)) {
1067                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1068                     &sc->sc_cpus);
1069                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1070         }
1071
1072         return (0);
1073 }
1074
1075 static int
1076 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1077 {
1078         struct gicv3_its_irqsrc *girq;
1079
1080         gicv3_its_select_cpu(dev, isrc);
1081
1082         girq = (struct gicv3_its_irqsrc *)isrc;
1083         its_cmd_movi(dev, girq);
1084         return (0);
1085 }
1086
1087 static int
1088 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1089     struct intr_irqsrc **isrcp)
1090 {
1091
1092         /*
1093          * This should never happen, we only call this function to map
1094          * interrupts found before the controller driver is ready.
1095          */
1096         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1097 }
1098
1099 static int
1100 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1101     struct resource *res, struct intr_map_data *data)
1102 {
1103
1104         /* Bind the interrupt to a CPU */
1105         gicv3_its_bind_intr(dev, isrc);
1106
1107         return (0);
1108 }
1109
1110 #ifdef SMP
1111 static void
1112 gicv3_its_init_secondary(device_t dev)
1113 {
1114         struct gicv3_its_softc *sc;
1115
1116         sc = device_get_softc(dev);
1117
1118         /*
1119          * This is fatal as otherwise we may bind interrupts to this CPU.
1120          * We need a way to tell the interrupt framework to only bind to a
1121          * subset of given CPUs when it performs the shuffle.
1122          */
1123         if (its_init_cpu(dev, sc) != 0)
1124                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1125                     PCPU_GET(cpuid));
1126 }
1127 #endif
1128
1129 static uint32_t
1130 its_get_devid(device_t pci_dev)
1131 {
1132         uintptr_t id;
1133
1134         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1135                 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1136                     device_get_nameunit(pci_dev));
1137
1138         return (id);
1139 }
1140
1141 static struct its_dev *
1142 its_device_find(device_t dev, device_t child)
1143 {
1144         struct gicv3_its_softc *sc;
1145         struct its_dev *its_dev = NULL;
1146
1147         sc = device_get_softc(dev);
1148
1149         mtx_lock_spin(&sc->sc_its_dev_lock);
1150         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1151                 if (its_dev->pci_dev == child)
1152                         break;
1153         }
1154         mtx_unlock_spin(&sc->sc_its_dev_lock);
1155
1156         return (its_dev);
1157 }
1158
1159 static struct its_dev *
1160 its_device_get(device_t dev, device_t child, u_int nvecs)
1161 {
1162         struct gicv3_its_softc *sc;
1163         struct its_dev *its_dev;
1164         vmem_addr_t irq_base;
1165         size_t esize;
1166
1167         sc = device_get_softc(dev);
1168
1169         its_dev = its_device_find(dev, child);
1170         if (its_dev != NULL)
1171                 return (its_dev);
1172
1173         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1174         if (its_dev == NULL)
1175                 return (NULL);
1176
1177         its_dev->pci_dev = child;
1178         its_dev->devid = its_get_devid(child);
1179
1180         its_dev->lpis.lpi_busy = 0;
1181         its_dev->lpis.lpi_num = nvecs;
1182         its_dev->lpis.lpi_free = nvecs;
1183
1184         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1185             &irq_base) != 0) {
1186                 free(its_dev, M_GICV3_ITS);
1187                 return (NULL);
1188         }
1189         its_dev->lpis.lpi_base = irq_base;
1190
1191         /* Get ITT entry size */
1192         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1193
1194         /*
1195          * Allocate ITT for this device.
1196          * PA has to be 256 B aligned. At least two entries for device.
1197          */
1198         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1199         its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
1200             M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1201             LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1202         if (its_dev->itt == 0) {
1203                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1204                 free(its_dev, M_GICV3_ITS);
1205                 return (NULL);
1206         }
1207
1208         /* Make sure device sees zeroed ITT. */
1209         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1210                 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1211
1212         mtx_lock_spin(&sc->sc_its_dev_lock);
1213         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1214         mtx_unlock_spin(&sc->sc_its_dev_lock);
1215
1216         /* Map device to its ITT */
1217         its_cmd_mapd(dev, its_dev, 1);
1218
1219         return (its_dev);
1220 }
1221
1222 static void
1223 its_device_release(device_t dev, struct its_dev *its_dev)
1224 {
1225         struct gicv3_its_softc *sc;
1226
1227         KASSERT(its_dev->lpis.lpi_busy == 0,
1228             ("its_device_release: Trying to release an inuse ITS device"));
1229
1230         /* Unmap device in ITS */
1231         its_cmd_mapd(dev, its_dev, 0);
1232
1233         sc = device_get_softc(dev);
1234
1235         /* Remove the device from the list of devices */
1236         mtx_lock_spin(&sc->sc_its_dev_lock);
1237         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1238         mtx_unlock_spin(&sc->sc_its_dev_lock);
1239
1240         /* Free ITT */
1241         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1242         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1243
1244         /* Free the IRQ allocation */
1245         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1246             its_dev->lpis.lpi_num);
1247
1248         free(its_dev, M_GICV3_ITS);
1249 }
1250
1251 static struct gicv3_its_irqsrc *
1252 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1253 {
1254         struct gicv3_its_irqsrc *girq = NULL;
1255
1256         KASSERT(sc->sc_irqs[irq] == NULL,
1257             ("%s: Interrupt %u already allocated", __func__, irq));
1258         mtx_lock_spin(&sc->sc_its_dev_lock);
1259         if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1260                 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1261                 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1262         }
1263         mtx_unlock_spin(&sc->sc_its_dev_lock);
1264         if (girq == NULL) {
1265                 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1266                     M_NOWAIT | M_ZERO);
1267                 if (girq == NULL)
1268                         return (NULL);
1269                 girq->gi_id = -1;
1270                 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1271                     "%s,%u", device_get_nameunit(dev), irq) != 0) {
1272                         free(girq, M_GICV3_ITS);
1273                         return (NULL);
1274                 }
1275         }
1276         girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1277         sc->sc_irqs[irq] = girq;
1278
1279         return (girq);
1280 }
1281
1282 static void
1283 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1284     struct gicv3_its_irqsrc *girq)
1285 {
1286         u_int irq;
1287
1288         mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1289
1290         irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1291         sc->sc_irqs[irq] = NULL;
1292
1293         girq->gi_id = -1;
1294         girq->gi_its_dev = NULL;
1295         TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1296 }
1297
1298 static int
1299 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1300     device_t *pic, struct intr_irqsrc **srcs)
1301 {
1302         struct gicv3_its_softc *sc;
1303         struct gicv3_its_irqsrc *girq;
1304         struct its_dev *its_dev;
1305         u_int irq;
1306         int i;
1307
1308         its_dev = its_device_get(dev, child, count);
1309         if (its_dev == NULL)
1310                 return (ENXIO);
1311
1312         KASSERT(its_dev->lpis.lpi_free >= count,
1313             ("gicv3_its_alloc_msi: No free LPIs"));
1314         sc = device_get_softc(dev);
1315         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1316             its_dev->lpis.lpi_free;
1317
1318         /* Allocate the irqsrc for each MSI */
1319         for (i = 0; i < count; i++, irq++) {
1320                 its_dev->lpis.lpi_free--;
1321                 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1322                     sc, irq);
1323                 if (srcs[i] == NULL)
1324                         break;
1325         }
1326
1327         /* The allocation failed, release them */
1328         if (i != count) {
1329                 mtx_lock_spin(&sc->sc_its_dev_lock);
1330                 for (i = 0; i < count; i++) {
1331                         girq = (struct gicv3_its_irqsrc *)srcs[i];
1332                         if (girq == NULL)
1333                                 break;
1334                         gicv3_its_release_irqsrc(sc, girq);
1335                         srcs[i] = NULL;
1336                 }
1337                 mtx_unlock_spin(&sc->sc_its_dev_lock);
1338                 return (ENXIO);
1339         }
1340
1341         /* Finish the allocation now we have all MSI irqsrcs */
1342         for (i = 0; i < count; i++) {
1343                 girq = (struct gicv3_its_irqsrc *)srcs[i];
1344                 girq->gi_id = i;
1345                 girq->gi_its_dev = its_dev;
1346
1347                 /* Map the message to the given IRQ */
1348                 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1349                 its_cmd_mapti(dev, girq);
1350         }
1351         its_dev->lpis.lpi_busy += count;
1352         *pic = dev;
1353
1354         return (0);
1355 }
1356
1357 static int
1358 gicv3_its_release_msi(device_t dev, device_t child, int count,
1359     struct intr_irqsrc **isrc)
1360 {
1361         struct gicv3_its_softc *sc;
1362         struct gicv3_its_irqsrc *girq;
1363         struct its_dev *its_dev;
1364         int i;
1365
1366         its_dev = its_device_find(dev, child);
1367
1368         KASSERT(its_dev != NULL,
1369             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1370              "no ITS device"));
1371         KASSERT(its_dev->lpis.lpi_busy >= count,
1372             ("gicv3_its_release_msi: Releasing more interrupts than "
1373              "were allocated: releasing %d, allocated %d", count,
1374              its_dev->lpis.lpi_busy));
1375
1376         sc = device_get_softc(dev);
1377         mtx_lock_spin(&sc->sc_its_dev_lock);
1378         for (i = 0; i < count; i++) {
1379                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1380                 gicv3_its_release_irqsrc(sc, girq);
1381         }
1382         mtx_unlock_spin(&sc->sc_its_dev_lock);
1383         its_dev->lpis.lpi_busy -= count;
1384
1385         if (its_dev->lpis.lpi_busy == 0)
1386                 its_device_release(dev, its_dev);
1387
1388         return (0);
1389 }
1390
1391 static int
1392 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1393     struct intr_irqsrc **isrcp)
1394 {
1395         struct gicv3_its_softc *sc;
1396         struct gicv3_its_irqsrc *girq;
1397         struct its_dev *its_dev;
1398         u_int nvecs, irq;
1399
1400         nvecs = pci_msix_count(child);
1401         its_dev = its_device_get(dev, child, nvecs);
1402         if (its_dev == NULL)
1403                 return (ENXIO);
1404
1405         KASSERT(its_dev->lpis.lpi_free > 0,
1406             ("gicv3_its_alloc_msix: No free LPIs"));
1407         sc = device_get_softc(dev);
1408         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1409             its_dev->lpis.lpi_free;
1410
1411         girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1412         if (girq == NULL)
1413                 return (ENXIO);
1414         girq->gi_id = its_dev->lpis.lpi_busy;
1415         girq->gi_its_dev = its_dev;
1416
1417         its_dev->lpis.lpi_free--;
1418         its_dev->lpis.lpi_busy++;
1419
1420         /* Map the message to the given IRQ */
1421         gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1422         its_cmd_mapti(dev, girq);
1423
1424         *pic = dev;
1425         *isrcp = (struct intr_irqsrc *)girq;
1426
1427         return (0);
1428 }
1429
1430 static int
1431 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1432 {
1433         struct gicv3_its_softc *sc;
1434         struct gicv3_its_irqsrc *girq;
1435         struct its_dev *its_dev;
1436
1437         its_dev = its_device_find(dev, child);
1438
1439         KASSERT(its_dev != NULL,
1440             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1441              "no ITS device"));
1442         KASSERT(its_dev->lpis.lpi_busy > 0,
1443             ("gicv3_its_release_msix: Releasing more interrupts than "
1444              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1445
1446         sc = device_get_softc(dev);
1447         girq = (struct gicv3_its_irqsrc *)isrc;
1448         mtx_lock_spin(&sc->sc_its_dev_lock);
1449         gicv3_its_release_irqsrc(sc, girq);
1450         mtx_unlock_spin(&sc->sc_its_dev_lock);
1451         its_dev->lpis.lpi_busy--;
1452
1453         if (its_dev->lpis.lpi_busy == 0)
1454                 its_device_release(dev, its_dev);
1455
1456         return (0);
1457 }
1458
1459 static int
1460 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1461     uint64_t *addr, uint32_t *data)
1462 {
1463         struct gicv3_its_softc *sc;
1464         struct gicv3_its_irqsrc *girq;
1465
1466         sc = device_get_softc(dev);
1467         girq = (struct gicv3_its_irqsrc *)isrc;
1468
1469         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1470         *data = girq->gi_id;
1471
1472         return (0);
1473 }
1474
1475 #ifdef IOMMU
1476 static int
1477 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1478 {
1479         struct gicv3_its_softc *sc;
1480         struct iommu_ctx *ctx;
1481         int error;
1482
1483         sc = device_get_softc(dev);
1484         ctx = iommu_get_dev_ctx(child);
1485         if (ctx == NULL)
1486                 return (ENXIO);
1487         /* Map the page containing the GITS_TRANSLATER register. */
1488         error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1489             IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1490         *domain = iommu_get_ctx_domain(ctx);
1491
1492         return (error);
1493 }
1494
1495 static void
1496 gicv3_iommu_deinit(device_t dev, device_t child)
1497 {
1498         struct iommu_ctx *ctx;
1499
1500         ctx = iommu_get_dev_ctx(child);
1501         if (ctx == NULL)
1502                 return;
1503
1504         iommu_unmap_msi(ctx);
1505 }
1506 #endif
1507
1508 /*
1509  * Commands handling.
1510  */
1511
1512 static __inline void
1513 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1514 {
1515         /* Command field: DW0 [7:0] */
1516         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1517         cmd->cmd_dword[0] |= htole64(cmd_type);
1518 }
1519
1520 static __inline void
1521 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1522 {
1523         /* Device ID field: DW0 [63:32] */
1524         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1525         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1526 }
1527
1528 static __inline void
1529 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1530 {
1531         /* Size field: DW1 [4:0] */
1532         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1533         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1534 }
1535
1536 static __inline void
1537 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1538 {
1539         /* ID field: DW1 [31:0] */
1540         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1541         cmd->cmd_dword[1] |= htole64(id);
1542 }
1543
1544 static __inline void
1545 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1546 {
1547         /* Physical ID field: DW1 [63:32] */
1548         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1549         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1550 }
1551
1552 static __inline void
1553 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1554 {
1555         /* Collection field: DW2 [16:0] */
1556         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1557         cmd->cmd_dword[2] |= htole64(col_id);
1558 }
1559
1560 static __inline void
1561 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1562 {
1563         /* Target Address field: DW2 [47:16] */
1564         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1565         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1566 }
1567
1568 static __inline void
1569 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1570 {
1571         /* ITT Address field: DW2 [47:8] */
1572         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1573         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1574 }
1575
1576 static __inline void
1577 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1578 {
1579         /* Valid field: DW2 [63] */
1580         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1581         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1582 }
1583
1584 static inline bool
1585 its_cmd_queue_full(struct gicv3_its_softc *sc)
1586 {
1587         size_t read_idx, next_write_idx;
1588
1589         /* Get the index of the next command */
1590         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1591             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1592         /* And the index of the current command being read */
1593         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1594
1595         /*
1596          * The queue is full when the write offset points
1597          * at the command before the current read offset.
1598          */
1599         return (next_write_idx == read_idx);
1600 }
1601
1602 static inline void
1603 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1604 {
1605
1606         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1607                 /* Clean D-cache under command. */
1608                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1609         } else {
1610                 /* DSB inner shareable, store */
1611                 dsb(ishst);
1612         }
1613
1614 }
1615
1616 static inline uint64_t
1617 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1618 {
1619         uint64_t off;
1620
1621         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1622
1623         return (off);
1624 }
1625
1626 static void
1627 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1628     struct its_cmd *cmd_last)
1629 {
1630         struct gicv3_its_softc *sc;
1631         uint64_t first, last, read;
1632         size_t us_left;
1633
1634         sc = device_get_softc(dev);
1635
1636         /*
1637          * XXX ARM64TODO: This is obviously a significant delay.
1638          * The reason for that is that currently the time frames for
1639          * the command to complete are not known.
1640          */
1641         us_left = 1000000;
1642
1643         first = its_cmd_cwriter_offset(sc, cmd_first);
1644         last = its_cmd_cwriter_offset(sc, cmd_last);
1645
1646         for (;;) {
1647                 read = gic_its_read_8(sc, GITS_CREADR);
1648                 if (first < last) {
1649                         if (read < first || read >= last)
1650                                 break;
1651                 } else if (read < first && read >= last)
1652                         break;
1653
1654                 if (us_left-- == 0) {
1655                         /* This means timeout */
1656                         device_printf(dev,
1657                             "Timeout while waiting for CMD completion.\n");
1658                         return;
1659                 }
1660                 DELAY(1);
1661         }
1662 }
1663
1664 static struct its_cmd *
1665 its_cmd_alloc_locked(device_t dev)
1666 {
1667         struct gicv3_its_softc *sc;
1668         struct its_cmd *cmd;
1669         size_t us_left;
1670
1671         sc = device_get_softc(dev);
1672
1673         /*
1674          * XXX ARM64TODO: This is obviously a significant delay.
1675          * The reason for that is that currently the time frames for
1676          * the command to complete (and therefore free the descriptor)
1677          * are not known.
1678          */
1679         us_left = 1000000;
1680
1681         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1682         while (its_cmd_queue_full(sc)) {
1683                 if (us_left-- == 0) {
1684                         /* Timeout while waiting for free command */
1685                         device_printf(dev,
1686                             "Timeout while waiting for free command\n");
1687                         return (NULL);
1688                 }
1689                 DELAY(1);
1690         }
1691
1692         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1693         sc->sc_its_cmd_next_idx++;
1694         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1695
1696         return (cmd);
1697 }
1698
1699 static uint64_t
1700 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1701 {
1702         uint64_t target;
1703         uint8_t cmd_type;
1704         u_int size;
1705
1706         cmd_type = desc->cmd_type;
1707         target = ITS_TARGET_NONE;
1708
1709         switch (cmd_type) {
1710         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1711                 target = desc->cmd_desc_movi.col->col_target;
1712                 cmd_format_command(cmd, ITS_CMD_MOVI);
1713                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1714                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1715                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1716                 break;
1717         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1718                 target = desc->cmd_desc_sync.col->col_target;
1719                 cmd_format_command(cmd, ITS_CMD_SYNC);
1720                 cmd_format_target(cmd, target);
1721                 break;
1722         case ITS_CMD_MAPD:      /* Assign ITT to device */
1723                 cmd_format_command(cmd, ITS_CMD_MAPD);
1724                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1725                 /*
1726                  * Size describes number of bits to encode interrupt IDs
1727                  * supported by the device minus one.
1728                  * When V (valid) bit is zero, this field should be written
1729                  * as zero.
1730                  */
1731                 if (desc->cmd_desc_mapd.valid != 0) {
1732                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1733                         size = MAX(1, size) - 1;
1734                 } else
1735                         size = 0;
1736
1737                 cmd_format_size(cmd, size);
1738                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1739                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1740                 break;
1741         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1742                 target = desc->cmd_desc_mapc.col->col_target;
1743                 cmd_format_command(cmd, ITS_CMD_MAPC);
1744                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1745                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1746                 cmd_format_target(cmd, target);
1747                 break;
1748         case ITS_CMD_MAPTI:
1749                 target = desc->cmd_desc_mapvi.col->col_target;
1750                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1751                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1752                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1753                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1754                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1755                 break;
1756         case ITS_CMD_MAPI:
1757                 target = desc->cmd_desc_mapi.col->col_target;
1758                 cmd_format_command(cmd, ITS_CMD_MAPI);
1759                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1760                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1761                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1762                 break;
1763         case ITS_CMD_INV:
1764                 target = desc->cmd_desc_inv.col->col_target;
1765                 cmd_format_command(cmd, ITS_CMD_INV);
1766                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1767                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1768                 break;
1769         case ITS_CMD_INVALL:
1770                 cmd_format_command(cmd, ITS_CMD_INVALL);
1771                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1772                 break;
1773         default:
1774                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1775         }
1776
1777         return (target);
1778 }
1779
1780 static int
1781 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1782 {
1783         struct gicv3_its_softc *sc;
1784         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1785         struct its_col col_sync;
1786         struct its_cmd_desc desc_sync;
1787         uint64_t target, cwriter;
1788
1789         sc = device_get_softc(dev);
1790         mtx_lock_spin(&sc->sc_its_cmd_lock);
1791         cmd = its_cmd_alloc_locked(dev);
1792         if (cmd == NULL) {
1793                 device_printf(dev, "could not allocate ITS command\n");
1794                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1795                 return (EBUSY);
1796         }
1797
1798         target = its_cmd_prepare(cmd, desc);
1799         its_cmd_sync(sc, cmd);
1800
1801         if (target != ITS_TARGET_NONE) {
1802                 cmd_sync = its_cmd_alloc_locked(dev);
1803                 if (cmd_sync != NULL) {
1804                         desc_sync.cmd_type = ITS_CMD_SYNC;
1805                         col_sync.col_target = target;
1806                         desc_sync.cmd_desc_sync.col = &col_sync;
1807                         its_cmd_prepare(cmd_sync, &desc_sync);
1808                         its_cmd_sync(sc, cmd_sync);
1809                 }
1810         }
1811
1812         /* Update GITS_CWRITER */
1813         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
1814         gic_its_write_8(sc, GITS_CWRITER, cwriter);
1815         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1816         mtx_unlock_spin(&sc->sc_its_cmd_lock);
1817
1818         its_cmd_wait_completion(dev, cmd, cmd_write);
1819
1820         return (0);
1821 }
1822
1823 /* Handlers to send commands */
1824 static void
1825 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
1826 {
1827         struct gicv3_its_softc *sc;
1828         struct its_cmd_desc desc;
1829         struct its_col *col;
1830
1831         sc = device_get_softc(dev);
1832         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1833
1834         desc.cmd_type = ITS_CMD_MOVI;
1835         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
1836         desc.cmd_desc_movi.col = col;
1837         desc.cmd_desc_movi.id = girq->gi_id;
1838
1839         its_cmd_send(dev, &desc);
1840 }
1841
1842 static void
1843 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
1844 {
1845         struct its_cmd_desc desc;
1846
1847         desc.cmd_type = ITS_CMD_MAPC;
1848         desc.cmd_desc_mapc.col = col;
1849         /*
1850          * Valid bit set - map the collection.
1851          * Valid bit cleared - unmap the collection.
1852          */
1853         desc.cmd_desc_mapc.valid = valid;
1854
1855         its_cmd_send(dev, &desc);
1856 }
1857
1858 static void
1859 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
1860 {
1861         struct gicv3_its_softc *sc;
1862         struct its_cmd_desc desc;
1863         struct its_col *col;
1864         u_int col_id;
1865
1866         sc = device_get_softc(dev);
1867
1868         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
1869         col = sc->sc_its_cols[col_id];
1870
1871         desc.cmd_type = ITS_CMD_MAPTI;
1872         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
1873         desc.cmd_desc_mapvi.col = col;
1874         /* The EventID sent to the device */
1875         desc.cmd_desc_mapvi.id = girq->gi_id;
1876         /* The physical interrupt presented to softeware */
1877         desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
1878
1879         its_cmd_send(dev, &desc);
1880 }
1881
1882 static void
1883 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
1884 {
1885         struct its_cmd_desc desc;
1886
1887         desc.cmd_type = ITS_CMD_MAPD;
1888         desc.cmd_desc_mapd.its_dev = its_dev;
1889         desc.cmd_desc_mapd.valid = valid;
1890
1891         its_cmd_send(dev, &desc);
1892 }
1893
1894 static void
1895 its_cmd_inv(device_t dev, struct its_dev *its_dev,
1896     struct gicv3_its_irqsrc *girq)
1897 {
1898         struct gicv3_its_softc *sc;
1899         struct its_cmd_desc desc;
1900         struct its_col *col;
1901
1902         sc = device_get_softc(dev);
1903         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1904
1905         desc.cmd_type = ITS_CMD_INV;
1906         /* The EventID sent to the device */
1907         desc.cmd_desc_inv.pid = girq->gi_id;
1908         desc.cmd_desc_inv.its_dev = its_dev;
1909         desc.cmd_desc_inv.col = col;
1910
1911         its_cmd_send(dev, &desc);
1912 }
1913
1914 static void
1915 its_cmd_invall(device_t dev, struct its_col *col)
1916 {
1917         struct its_cmd_desc desc;
1918
1919         desc.cmd_type = ITS_CMD_INVALL;
1920         desc.cmd_desc_invall.col = col;
1921
1922         its_cmd_send(dev, &desc);
1923 }
1924
1925 #ifdef FDT
1926 static device_probe_t gicv3_its_fdt_probe;
1927 static device_attach_t gicv3_its_fdt_attach;
1928
1929 static device_method_t gicv3_its_fdt_methods[] = {
1930         /* Device interface */
1931         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
1932         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
1933
1934         /* End */
1935         DEVMETHOD_END
1936 };
1937
1938 #define its_baseclasses its_fdt_baseclasses
1939 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
1940     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1941 #undef its_baseclasses
1942
1943 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
1944     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1945
1946 static int
1947 gicv3_its_fdt_probe(device_t dev)
1948 {
1949
1950         if (!ofw_bus_status_okay(dev))
1951                 return (ENXIO);
1952
1953         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
1954                 return (ENXIO);
1955
1956         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
1957         return (BUS_PROBE_DEFAULT);
1958 }
1959
1960 static int
1961 gicv3_its_fdt_attach(device_t dev)
1962 {
1963         struct gicv3_its_softc *sc;
1964         phandle_t xref;
1965         int err;
1966
1967         sc = device_get_softc(dev);
1968         sc->dev = dev;
1969         err = gicv3_its_attach(dev);
1970         if (err != 0)
1971                 return (err);
1972
1973         /* Register this device as a interrupt controller */
1974         xref = OF_xref_from_node(ofw_bus_get_node(dev));
1975         sc->sc_pic = intr_pic_register(dev, xref);
1976         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
1977             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
1978         if (err != 0) {
1979                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
1980                 return (err);
1981         }
1982
1983         /* Register this device to handle MSI interrupts */
1984         err = intr_msi_register(dev, xref);
1985         if (err != 0) {
1986                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
1987                 return (err);
1988         }
1989
1990         return (0);
1991 }
1992 #endif
1993
1994 #ifdef DEV_ACPI
1995 static device_probe_t gicv3_its_acpi_probe;
1996 static device_attach_t gicv3_its_acpi_attach;
1997
1998 static device_method_t gicv3_its_acpi_methods[] = {
1999         /* Device interface */
2000         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
2001         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
2002
2003         /* End */
2004         DEVMETHOD_END
2005 };
2006
2007 #define its_baseclasses its_acpi_baseclasses
2008 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2009     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2010 #undef its_baseclasses
2011
2012 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2013     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2014
2015 static int
2016 gicv3_its_acpi_probe(device_t dev)
2017 {
2018
2019         if (gic_get_bus(dev) != GIC_BUS_ACPI)
2020                 return (EINVAL);
2021
2022         if (gic_get_hw_rev(dev) < 3)
2023                 return (EINVAL);
2024
2025         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2026         return (BUS_PROBE_DEFAULT);
2027 }
2028
2029 static int
2030 gicv3_its_acpi_attach(device_t dev)
2031 {
2032         struct gicv3_its_softc *sc;
2033         struct gic_v3_devinfo *di;
2034         int err;
2035
2036         sc = device_get_softc(dev);
2037         sc->dev = dev;
2038         err = gicv3_its_attach(dev);
2039         if (err != 0)
2040                 return (err);
2041
2042         di = device_get_ivars(dev);
2043         sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2044         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2045             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2046         if (err != 0) {
2047                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2048                 return (err);
2049         }
2050
2051         /* Register this device to handle MSI interrupts */
2052         err = intr_msi_register(dev, di->msi_xref);
2053         if (err != 0) {
2054                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2055                 return (err);
2056         }
2057
2058         return (0);
2059 }
2060 #endif