sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  *
   4  * This software was developed by Andrew Turner under
   5  * the sponsorship of the FreeBSD Foundation.
   6  *
   7  * This software was developed by Semihalf under
   8  * the sponsorship of the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include "opt_acpi.h"
  33 #include "opt_platform.h"
  34 #include "opt_iommu.h"
  35
  36 #include <sys/cdefs.h>
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/bus.h>
  40 #include <sys/cpuset.h>
  41 #include <sys/domainset.h>
  42 #include <sys/endian.h>
  43 #include <sys/kernel.h>
  44 #include <sys/lock.h>
  45 #include <sys/malloc.h>
  46 #include <sys/module.h>
  47 #include <sys/mutex.h>
  48 #include <sys/proc.h>
  49 #include <sys/taskqueue.h>
  50 #include <sys/tree.h>
  51 #include <sys/queue.h>
  52 #include <sys/rman.h>
  53 #include <sys/sbuf.h>
  54 #include <sys/smp.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/vmem.h>
  57
  58 #include <vm/vm.h>
  59 #include <vm/pmap.h>
  60 #include <vm/vm_page.h>
  61
  62 #include <machine/bus.h>
  63 #include <machine/intr.h>
  64
  65 #include <arm/arm/gic_common.h>
  66 #include <arm64/arm64/gic_v3_reg.h>
  67 #include <arm64/arm64/gic_v3_var.h>
  68
  69 #ifdef FDT
  70 #include <dev/ofw/openfirm.h>
  71 #include <dev/ofw/ofw_bus.h>
  72 #include <dev/ofw/ofw_bus_subr.h>
  73 #endif
  74 #include <dev/pci/pcireg.h>
  75 #include <dev/pci/pcivar.h>
  76
  77 #ifdef IOMMU
  78 #include <dev/iommu/iommu.h>
  79 #include <dev/iommu/iommu_gas.h>
  80 #endif
  81
  82 #include "pcib_if.h"
  83 #include "pic_if.h"
  84 #include "msi_if.h"
  85
  86 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  87     "ARM GICv3 Interrupt Translation Service");
  88
  89 #define LPI_NIRQS               (64 * 1024)
  90
  91 /* The size and alignment of the command circular buffer */
  92 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  93 #define ITS_CMDQ_ALIGN          (64 * 1024)
  94
  95 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  96 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  97 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  98
  99 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
 100 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
 101 #define LPI_PENDTAB_ALIGN       (64 * 1024)
 102 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
 103
 104 #define LPI_INT_TRANS_TAB_ALIGN 256
 105 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
 106
 107 /* ITS commands encoding */
 108 #define ITS_CMD_MOVI            (0x01)
 109 #define ITS_CMD_SYNC            (0x05)
 110 #define ITS_CMD_MAPD            (0x08)
 111 #define ITS_CMD_MAPC            (0x09)
 112 #define ITS_CMD_MAPTI           (0x0a)
 113 #define ITS_CMD_MAPI            (0x0b)
 114 #define ITS_CMD_INV             (0x0c)
 115 #define ITS_CMD_INVALL          (0x0d)
 116 /* Command */
 117 #define CMD_COMMAND_MASK        (0xFFUL)
 118 /* PCI device ID */
 119 #define CMD_DEVID_SHIFT         (32)
 120 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 121 /* Size of IRQ ID bitfield */
 122 #define CMD_SIZE_MASK           (0xFFUL)
 123 /* Virtual LPI ID */
 124 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 125 /* Physical LPI ID */
 126 #define CMD_PID_SHIFT           (32)
 127 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 128 /* Collection */
 129 #define CMD_COL_MASK            (0xFFFFUL)
 130 /* Target (CPU or Re-Distributor) */
 131 #define CMD_TARGET_SHIFT        (16)
 132 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 133 /* Interrupt Translation Table address */
 134 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 135 /* Valid command bit */
 136 #define CMD_VALID_SHIFT         (63)
 137 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 138
 139 #define ITS_TARGET_NONE         0xFBADBEEF
 140
 141 /* LPI chunk owned by ITS device */
 142 struct lpi_chunk {
 143         u_int   lpi_base;
 144         u_int   lpi_free;       /* First free LPI in set */
 145         u_int   lpi_num;        /* Total number of LPIs in chunk */
 146         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 147 };
 148
 149 /* ITS device */
 150 struct its_dev {
 151         TAILQ_ENTRY(its_dev)    entry;
 152         /* PCI device */
 153         device_t                pci_dev;
 154         /* Device ID (i.e. PCI device ID) */
 155         uint32_t                devid;
 156         /* List of assigned LPIs */
 157         struct lpi_chunk        lpis;
 158         /* Virtual address of ITT */
 159         vm_offset_t             itt;
 160         size_t                  itt_size;
 161 };
 162
 163 /*
 164  * ITS command descriptor.
 165  * Idea for command description passing taken from Linux.
 166  */
 167 struct its_cmd_desc {
 168         uint8_t cmd_type;
 169
 170         union {
 171                 struct {
 172                         struct its_dev *its_dev;
 173                         struct its_col *col;
 174                         uint32_t id;
 175                 } cmd_desc_movi;
 176
 177                 struct {
 178                         struct its_col *col;
 179                 } cmd_desc_sync;
 180
 181                 struct {
 182                         struct its_col *col;
 183                         uint8_t valid;
 184                 } cmd_desc_mapc;
 185
 186                 struct {
 187                         struct its_dev *its_dev;
 188                         struct its_col *col;
 189                         uint32_t pid;
 190                         uint32_t id;
 191                 } cmd_desc_mapvi;
 192
 193                 struct {
 194                         struct its_dev *its_dev;
 195                         struct its_col *col;
 196                         uint32_t pid;
 197                 } cmd_desc_mapi;
 198
 199                 struct {
 200                         struct its_dev *its_dev;
 201                         uint8_t valid;
 202                 } cmd_desc_mapd;
 203
 204                 struct {
 205                         struct its_dev *its_dev;
 206                         struct its_col *col;
 207                         uint32_t pid;
 208                 } cmd_desc_inv;
 209
 210                 struct {
 211                         struct its_col *col;
 212                 } cmd_desc_invall;
 213         };
 214 };
 215
 216 /* ITS command. Each command is 32 bytes long */
 217 struct its_cmd {
 218         uint64_t        cmd_dword[4];   /* ITS command double word */
 219 };
 220
 221 /* An ITS private table */
 222 struct its_ptable {
 223         vm_offset_t     ptab_vaddr;
 224         unsigned long   ptab_size;
 225 };
 226
 227 /* ITS collection description. */
 228 struct its_col {
 229         uint64_t        col_target;     /* Target Re-Distributor */
 230         uint64_t        col_id;         /* Collection ID */
 231 };
 232
 233 struct gicv3_its_irqsrc {
 234         struct intr_irqsrc      gi_isrc;
 235         u_int                   gi_id;
 236         u_int                   gi_lpi;
 237         struct its_dev          *gi_its_dev;
 238         TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
 239 };
 240
 241 struct gicv3_its_softc {
 242         device_t        dev;
 243         struct intr_pic *sc_pic;
 244         struct resource *sc_its_res;
 245
 246         cpuset_t        sc_cpus;
 247         struct domainset *sc_ds;
 248         u_int           gic_irq_cpu;
 249
 250         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 251         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 252
 253         /*
 254          * TODO: We should get these from the parent as we only want a
 255          * single copy of each across the interrupt controller.
 256          */
 257         uint8_t         *sc_conf_base;
 258         vm_offset_t sc_pend_base[MAXCPU];
 259
 260         /* Command handling */
 261         struct mtx sc_its_cmd_lock;
 262         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 263         size_t sc_its_cmd_next_idx;
 264
 265         vmem_t *sc_irq_alloc;
 266         struct gicv3_its_irqsrc **sc_irqs;
 267         u_int   sc_irq_base;
 268         u_int   sc_irq_length;
 269         u_int   sc_irq_count;
 270
 271         struct mtx sc_its_dev_lock;
 272         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 273         TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
 274
 275 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 276 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 277 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 278         u_int sc_its_flags;
 279         bool    trace_enable;
 280         vm_page_t ma; /* fake msi page */
 281 };
 282
 283 static void *conf_base;
 284
 285 typedef void (its_quirk_func_t)(device_t);
 286 static its_quirk_func_t its_quirk_cavium_22375;
 287
 288 static const struct {
 289         const char *desc;
 290         uint32_t iidr;
 291         uint32_t iidr_mask;
 292         its_quirk_func_t *func;
 293 } its_quirks[] = {
 294         {
 295                 /* Cavium ThunderX Pass 1.x */
 296                 .desc = "Cavium ThunderX errata: 22375, 24313",
 297                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 298                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 299                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 300                 .func = its_quirk_cavium_22375,
 301         },
 302 };
 303
 304 #define gic_its_read_4(sc, reg)                 \
 305     bus_read_4((sc)->sc_its_res, (reg))
 306 #define gic_its_read_8(sc, reg)                 \
 307     bus_read_8((sc)->sc_its_res, (reg))
 308
 309 #define gic_its_write_4(sc, reg, val)           \
 310     bus_write_4((sc)->sc_its_res, (reg), (val))
 311 #define gic_its_write_8(sc, reg, val)           \
 312     bus_write_8((sc)->sc_its_res, (reg), (val))
 313
 314 static device_attach_t gicv3_its_attach;
 315 static device_detach_t gicv3_its_detach;
 316
 317 static pic_disable_intr_t gicv3_its_disable_intr;
 318 static pic_enable_intr_t gicv3_its_enable_intr;
 319 static pic_map_intr_t gicv3_its_map_intr;
 320 static pic_setup_intr_t gicv3_its_setup_intr;
 321 static pic_post_filter_t gicv3_its_post_filter;
 322 static pic_post_ithread_t gicv3_its_post_ithread;
 323 static pic_pre_ithread_t gicv3_its_pre_ithread;
 324 static pic_bind_intr_t gicv3_its_bind_intr;
 325 #ifdef SMP
 326 static pic_init_secondary_t gicv3_its_init_secondary;
 327 #endif
 328 static msi_alloc_msi_t gicv3_its_alloc_msi;
 329 static msi_release_msi_t gicv3_its_release_msi;
 330 static msi_alloc_msix_t gicv3_its_alloc_msix;
 331 static msi_release_msix_t gicv3_its_release_msix;
 332 static msi_map_msi_t gicv3_its_map_msi;
 333 #ifdef IOMMU
 334 static msi_iommu_init_t gicv3_iommu_init;
 335 static msi_iommu_deinit_t gicv3_iommu_deinit;
 336 #endif
 337
 338 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 339 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 340 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 341 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 342 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 343 static void its_cmd_invall(device_t, struct its_col *);
 344
 345 static device_method_t gicv3_its_methods[] = {
 346         /* Device interface */
 347         DEVMETHOD(device_detach,        gicv3_its_detach),
 348
 349         /* Interrupt controller interface */
 350         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 351         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 352         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 353         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 354         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 355         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 356         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 357 #ifdef SMP
 358         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 359         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 360 #endif
 361
 362         /* MSI/MSI-X */
 363         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 364         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 365         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 366         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 367         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 368 #ifdef IOMMU
 369         DEVMETHOD(msi_iommu_init,       gicv3_iommu_init),
 370         DEVMETHOD(msi_iommu_deinit,     gicv3_iommu_deinit),
 371 #endif
 372
 373         /* End */
 374         DEVMETHOD_END
 375 };
 376
 377 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 378     sizeof(struct gicv3_its_softc));
 379
 380 static void
 381 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 382 {
 383         vm_paddr_t cmd_paddr;
 384         uint64_t reg, tmp;
 385
 386         /* Set up the command circular buffer */
 387         sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
 388             sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
 389             0);
 390         sc->sc_its_cmd_next_idx = 0;
 391
 392         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 393
 394         /* Set the base of the command buffer */
 395         reg = GITS_CBASER_VALID |
 396             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 397             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 398             (ITS_CMDQ_SIZE / 4096 - 1);
 399         gic_its_write_8(sc, GITS_CBASER, reg);
 400
 401         /* Read back to check for fixed value fields */
 402         tmp = gic_its_read_8(sc, GITS_CBASER);
 403
 404         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 405             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 406                 /* Check if the hardware reported non-shareable */
 407                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 408                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 409                         /* If so remove the cache attribute */
 410                         reg &= ~GITS_CBASER_CACHE_MASK;
 411                         reg &= ~GITS_CBASER_SHARE_MASK;
 412                         /* Set to Non-cacheable, Non-shareable */
 413                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 414                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 415
 416                         gic_its_write_8(sc, GITS_CBASER, reg);
 417                 }
 418
 419                 /* The command queue has to be flushed after each command */
 420                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 421         }
 422
 423         /* Get the next command from the start of the buffer */
 424         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 425 }
 426
 427 static int
 428 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
 429 {
 430         uint64_t reg, tmp;
 431         int page_size;
 432
 433         page_size = PAGE_SIZE_64K;
 434         reg = gic_its_read_8(sc, GITS_BASER(table));
 435
 436         while (1) {
 437                 reg &= GITS_BASER_PSZ_MASK;
 438                 switch (page_size) {
 439                 case PAGE_SIZE_4K:      /* 4KB */
 440                         reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 441                         break;
 442                 case PAGE_SIZE_16K:     /* 16KB */
 443                         reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 444                         break;
 445                 case PAGE_SIZE_64K:     /* 64KB */
 446                         reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 447                         break;
 448                 }
 449
 450                 /* Write the new page size */
 451                 gic_its_write_8(sc, GITS_BASER(table), reg);
 452
 453                 /* Read back to check */
 454                 tmp = gic_its_read_8(sc, GITS_BASER(table));
 455
 456                 /* The page size is correct */
 457                 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
 458                         return (page_size);
 459
 460                 switch (page_size) {
 461                 default:
 462                         return (-1);
 463                 case PAGE_SIZE_16K:
 464                         page_size = PAGE_SIZE_4K;
 465                         break;
 466                 case PAGE_SIZE_64K:
 467                         page_size = PAGE_SIZE_16K;
 468                         break;
 469                 }
 470         }
 471 }
 472
 473 static int
 474 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 475 {
 476         vm_offset_t table;
 477         vm_paddr_t paddr;
 478         uint64_t cache, reg, share, tmp, type;
 479         size_t esize, its_tbl_size, nidents, nitspages, npages;
 480         int i, page_size;
 481         int devbits;
 482
 483         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 484                 /*
 485                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 486                  * are to be 21 bits in length. The entry size of the ITS
 487                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 488                  * is supposed to be 8 bytes in length (for device table).
 489                  * Finally the page size that is to be used by ITS to access
 490                  * this table will be set to 64KB.
 491                  *
 492                  * This gives 0x200000 entries of size 0x8 bytes covered by
 493                  * 256 pages each of which 64KB in size. The number of pages
 494                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 495                  * that case this value would be 0xFF but on ThunderX the
 496                  * maximum value that HW accepts is 0xFD.
 497                  *
 498                  * Set an arbitrary number of device ID bits to 20 in order
 499                  * to limit the number of entries in ITS device table to
 500                  * 0x100000 and the table size to 8MB.
 501                  */
 502                 devbits = 20;
 503                 cache = 0;
 504         } else {
 505                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 506                 cache = GITS_BASER_CACHE_WAWB;
 507         }
 508         share = GITS_BASER_SHARE_IS;
 509
 510         for (i = 0; i < GITS_BASER_NUM; i++) {
 511                 reg = gic_its_read_8(sc, GITS_BASER(i));
 512                 /* The type of table */
 513                 type = GITS_BASER_TYPE(reg);
 514                 if (type == GITS_BASER_TYPE_UNIMPL)
 515                         continue;
 516
 517                 /* The table entry size */
 518                 esize = GITS_BASER_ESIZE(reg);
 519
 520                 /* Find the tables page size */
 521                 page_size = gicv3_its_table_page_size(sc, i);
 522                 if (page_size == -1) {
 523                         device_printf(dev, "No valid page size for table %d\n",
 524                             i);
 525                         return (EINVAL);
 526                 }
 527
 528                 switch(type) {
 529                 case GITS_BASER_TYPE_DEV:
 530                         nidents = (1 << devbits);
 531                         its_tbl_size = esize * nidents;
 532                         its_tbl_size = roundup2(its_tbl_size, page_size);
 533                         break;
 534                 case GITS_BASER_TYPE_VP:
 535                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 536                 case GITS_BASER_TYPE_IC:
 537                         its_tbl_size = page_size;
 538                         break;
 539                 default:
 540                         if (bootverbose)
 541                                 device_printf(dev, "Unhandled table type %lx\n",
 542                                     type);
 543                         continue;
 544                 }
 545                 npages = howmany(its_tbl_size, PAGE_SIZE);
 546
 547                 /* Allocate the table */
 548                 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
 549                     M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
 550                     (1ul << 48) - 1, PAGE_SIZE_64K, 0);
 551
 552                 sc->sc_its_ptab[i].ptab_vaddr = table;
 553                 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;
 554
 555                 paddr = vtophys(table);
 556
 557                 while (1) {
 558                         nitspages = howmany(its_tbl_size, page_size);
 559
 560                         /* Clear the fields we will be setting */
 561                         reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
 562                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 563                             GITS_BASER_PA_MASK |
 564                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 565                             GITS_BASER_SIZE_MASK);
 566                         /* Set the new values */
 567                         reg |= GITS_BASER_VALID |
 568                             (cache << GITS_BASER_CACHE_SHIFT) |
 569                             (type << GITS_BASER_TYPE_SHIFT) |
 570                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 571                             (nitspages - 1);
 572
 573                         switch (page_size) {
 574                         case PAGE_SIZE_4K:      /* 4KB */
 575                                 reg |=
 576                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 577                                 break;
 578                         case PAGE_SIZE_16K:     /* 16KB */
 579                                 reg |=
 580                                     GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
 581                                 break;
 582                         case PAGE_SIZE_64K:     /* 64KB */
 583                                 reg |=
 584                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 585                                 break;
 586                         }
 587
 588                         gic_its_write_8(sc, GITS_BASER(i), reg);
 589
 590                         /* Read back to check */
 591                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 592
 593                         /* Do the shareability masks line up? */
 594                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 595                             (reg & GITS_BASER_SHARE_MASK)) {
 596                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 597                                     GITS_BASER_SHARE_SHIFT;
 598                                 continue;
 599                         }
 600
 601                         if (tmp != reg) {
 602                                 device_printf(dev, "GITS_BASER%d: "
 603                                     "unable to be updated: %lx != %lx\n",
 604                                     i, reg, tmp);
 605                                 return (ENXIO);
 606                         }
 607
 608                         /* We should have made all needed changes */
 609                         break;
 610                 }
 611         }
 612
 613         return (0);
 614 }
 615
 616 static void
 617 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 618 {
 619         void *conf_table;
 620
 621         conf_table = atomic_load_ptr(&conf_base);
 622         if (conf_table == NULL) {
 623                 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
 624                     M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
 625                     LPI_CONFTAB_ALIGN, 0);
 626
 627                 if (atomic_cmpset_ptr((uintptr_t *)&conf_base,
 628                     (uintptr_t)NULL, (uintptr_t)conf_table) == 0) {
 629                         contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS);
 630                         conf_table = atomic_load_ptr(&conf_base);
 631                 }
 632         }
 633         sc->sc_conf_base = conf_table;
 634
 635         /* Set the default configuration */
 636         memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 637             LPI_CONFTAB_SIZE);
 638
 639         /* Flush the table to memory */
 640         cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
 641 }
 642
 643 static void
 644 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 645 {
 646         int i;
 647
 648         for (i = 0; i <= mp_maxid; i++) {
 649                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 650                         continue;
 651
 652                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 653                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 654                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 655
 656                 /* Flush so the ITS can see the memory */
 657                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
 658                     LPI_PENDTAB_SIZE);
 659         }
 660 }
 661
 662 static void
 663 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
 664 {
 665         device_t gicv3;
 666         uint64_t xbaser, tmp;
 667         uint32_t ctlr;
 668         u_int cpuid;
 669
 670         gicv3 = device_get_parent(dev);
 671         cpuid = PCPU_GET(cpuid);
 672
 673         /* Disable LPIs */
 674         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 675         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 676         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 677
 678         /* Make sure changes are observable my the GIC */
 679         dsb(sy);
 680
 681         /*
 682          * Set the redistributor base
 683          */
 684         xbaser = vtophys(sc->sc_conf_base) |
 685             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 686             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 687             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 688         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 689
 690         /* Check the cache attributes we set */
 691         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 692
 693         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 694             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 695                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 696                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 697                         /* We need to mark as non-cacheable */
 698                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 699                             GICR_PROPBASER_CACHE_MASK);
 700                         /* Non-cacheable */
 701                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 702                             GICR_PROPBASER_CACHE_SHIFT;
 703                         /* Non-shareable */
 704                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 705                             GICR_PROPBASER_SHARE_SHIFT;
 706                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 707                 }
 708                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 709         }
 710
 711         /*
 712          * Set the LPI pending table base
 713          */
 714         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 715             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 716             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 717
 718         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 719
 720         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 721
 722         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 723             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 724                 /* Clear the cahce and shareability bits */
 725                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 726                     GICR_PENDBASER_SHARE_MASK);
 727                 /* Mark as non-shareable */
 728                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 729                 /* And non-cacheable */
 730                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 731                     GICR_PENDBASER_CACHE_SHIFT;
 732         }
 733
 734         /* Enable LPIs */
 735         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 736         ctlr |= GICR_CTLR_LPI_ENABLE;
 737         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 738
 739         /* Make sure the GIC has seen everything */
 740         dsb(sy);
 741 }
 742
 743 static int
 744 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 745 {
 746         device_t gicv3;
 747         vm_paddr_t target;
 748         u_int cpuid;
 749         struct redist_pcpu *rpcpu;
 750
 751         gicv3 = device_get_parent(dev);
 752         cpuid = PCPU_GET(cpuid);
 753         if (!CPU_ISSET(cpuid, &sc->sc_cpus))
 754                 return (0);
 755
 756         /* Check if the ITS is enabled on this CPU */
 757         if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
 758                 return (ENXIO);
 759
 760         rpcpu = gicv3_get_redist(dev);
 761
 762         /* Do per-cpu LPI init once */
 763         if (!rpcpu->lpi_enabled) {
 764                 its_init_cpu_lpi(dev, sc);
 765                 rpcpu->lpi_enabled = true;
 766         }
 767
 768         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 769                 /* This ITS wants the redistributor physical address */
 770                 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
 771                     rpcpu->offset);
 772         } else {
 773                 /* This ITS wants the unique processor number */
 774                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
 775                     CMD_TARGET_SHIFT;
 776         }
 777
 778         sc->sc_its_cols[cpuid]->col_target = target;
 779         sc->sc_its_cols[cpuid]->col_id = cpuid;
 780
 781         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 782         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 783
 784         return (0);
 785 }
 786
 787 static int
 788 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
 789 {
 790         struct gicv3_its_softc *sc;
 791         int rv;
 792
 793         sc = arg1;
 794
 795         rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
 796         if (rv != 0 || req->newptr == NULL)
 797                 return (rv);
 798         if (sc->trace_enable)
 799                 gic_its_write_8(sc, GITS_TRKCTLR, 3);
 800         else
 801                 gic_its_write_8(sc, GITS_TRKCTLR, 0);
 802
 803         return (0);
 804 }
 805
 806 static int
 807 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
 808 {
 809         struct gicv3_its_softc *sc;
 810         struct sbuf *sb;
 811         int err;
 812
 813         sc = arg1;
 814         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 815         if (sb == NULL) {
 816                 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
 817                 return (ENOMEM);
 818         }
 819         sbuf_cat(sb, "\n");
 820         sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
 821             gic_its_read_4(sc, GITS_TRKCTLR));
 822         sbuf_printf(sb, "GITS_TRKR:    0x%08X\n",
 823             gic_its_read_4(sc, GITS_TRKR));
 824         sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
 825             gic_its_read_4(sc, GITS_TRKDIDR));
 826         sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
 827             gic_its_read_4(sc, GITS_TRKPIDR));
 828         sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
 829             gic_its_read_4(sc, GITS_TRKVIDR));
 830         sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
 831            gic_its_read_4(sc, GITS_TRKTGTR));
 832
 833         err = sbuf_finish(sb);
 834         if (err)
 835                 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
 836         sbuf_delete(sb);
 837         return(err);
 838 }
 839
 840 static int
 841 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
 842 {
 843         struct sysctl_oid *oid, *child;
 844         struct sysctl_ctx_list *ctx_list;
 845
 846         ctx_list = device_get_sysctl_ctx(sc->dev);
 847         child = device_get_sysctl_tree(sc->dev);
 848         oid = SYSCTL_ADD_NODE(ctx_list,
 849             SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
 850             CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
 851         if (oid == NULL)
 852                 return (ENXIO);
 853
 854         /* Add registers */
 855         SYSCTL_ADD_PROC(ctx_list,
 856             SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
 857             CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 858             gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
 859         SYSCTL_ADD_PROC(ctx_list,
 860             SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
 861             CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
 862             gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
 863
 864         return (0);
 865 }
 866
 867 static int
 868 gicv3_its_attach(device_t dev)
 869 {
 870         struct gicv3_its_softc *sc;
 871         int domain, err, i, rid;
 872         uint64_t phys;
 873         uint32_t ctlr, iidr;
 874
 875         sc = device_get_softc(dev);
 876
 877         sc->sc_irq_length = gicv3_get_nirqs(dev);
 878         sc->sc_irq_base = GIC_FIRST_LPI;
 879         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
 880
 881         rid = 0;
 882         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 883             RF_ACTIVE);
 884         if (sc->sc_its_res == NULL) {
 885                 device_printf(dev, "Could not allocate memory\n");
 886                 return (ENXIO);
 887         }
 888
 889         phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
 890             GITS_TRANSLATER, PAGE_SIZE);
 891         sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
 892         vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
 893
 894         CPU_COPY(&all_cpus, &sc->sc_cpus);
 895         iidr = gic_its_read_4(sc, GITS_IIDR);
 896         for (i = 0; i < nitems(its_quirks); i++) {
 897                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 898                         if (bootverbose) {
 899                                 device_printf(dev, "Applying %s\n",
 900                                     its_quirks[i].desc);
 901                         }
 902                         its_quirks[i].func(dev);
 903                         break;
 904                 }
 905         }
 906
 907         if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
 908                 sc->sc_ds = DOMAINSET_PREF(domain);
 909         } else {
 910                 sc->sc_ds = DOMAINSET_RR();
 911         }
 912
 913         /*
 914          * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
 915          * coming in via, for instance, a kexec/kboot style setup where a
 916          * previous kernel has configured then relinquished control.  Clear it
 917          * so that we can reconfigure GITS_BASER*.
 918          */
 919         ctlr = gic_its_read_4(sc, GITS_CTLR);
 920         if ((ctlr & GITS_CTLR_EN) != 0) {
 921                 ctlr &= ~GITS_CTLR_EN;
 922                 gic_its_write_4(sc, GITS_CTLR, ctlr);
 923         }
 924
 925         /* Allocate the private tables */
 926         err = gicv3_its_table_init(dev, sc);
 927         if (err != 0)
 928                 return (err);
 929
 930         /* Protects access to the device list */
 931         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 932
 933         /* Protects access to the ITS command circular buffer. */
 934         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
 935
 936         /* Allocate the command circular buffer */
 937         gicv3_its_cmdq_init(sc);
 938
 939         /* Allocate the per-CPU collections */
 940         for (int cpu = 0; cpu <= mp_maxid; cpu++)
 941                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
 942                         sc->sc_its_cols[cpu] = malloc_domainset(
 943                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
 944                             DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
 945                             M_WAITOK | M_ZERO);
 946                 else
 947                         sc->sc_its_cols[cpu] = NULL;
 948
 949         /* Enable the ITS */
 950         gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
 951
 952         /* Create the LPI configuration table */
 953         gicv3_its_conftable_init(sc);
 954
 955         /* And the pending tebles */
 956         gicv3_its_pendtables_init(sc);
 957
 958         /* Enable LPIs on this CPU */
 959         its_init_cpu(dev, sc);
 960
 961         TAILQ_INIT(&sc->sc_its_dev_list);
 962         TAILQ_INIT(&sc->sc_free_irqs);
 963
 964         /*
 965          * Create the vmem object to allocate INTRNG IRQs from. We try to
 966          * use all IRQs not already used by the GICv3.
 967          * XXX: This assumes there are no other interrupt controllers in the
 968          * system.
 969          */
 970         sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
 971             gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
 972
 973         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
 974             M_GICV3_ITS, M_WAITOK | M_ZERO);
 975
 976         /* For GIC-500 install tracking sysctls. */
 977         if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
 978             GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
 979                 gicv3_its_init_sysctl(sc);
 980
 981         return (0);
 982 }
 983
 984 static int
 985 gicv3_its_detach(device_t dev)
 986 {
 987
 988         return (ENXIO);
 989 }
 990
 991 static void
 992 its_quirk_cavium_22375(device_t dev)
 993 {
 994         struct gicv3_its_softc *sc;
 995         int domain;
 996
 997         sc = device_get_softc(dev);
 998         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
 999
1000         /*
1001          * We need to limit which CPUs we send these interrupts to on
1002          * the original dual socket ThunderX as it is unable to
1003          * forward them between the two sockets.
1004          */
1005         if (bus_get_domain(dev, &domain) == 0) {
1006                 if (domain < MAXMEMDOM) {
1007                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1008                 } else {
1009                         CPU_ZERO(&sc->sc_cpus);
1010                 }
1011         }
1012 }
1013
1014 static void
1015 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1016 {
1017         struct gicv3_its_softc *sc;
1018         struct gicv3_its_irqsrc *girq;
1019         uint8_t *conf;
1020
1021         sc = device_get_softc(dev);
1022         girq = (struct gicv3_its_irqsrc *)isrc;
1023         conf = sc->sc_conf_base;
1024
1025         conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1026
1027         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1028                 /* Clean D-cache under command. */
1029                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1030         } else {
1031                 /* DSB inner shareable, store */
1032                 dsb(ishst);
1033         }
1034
1035         its_cmd_inv(dev, girq->gi_its_dev, girq);
1036 }
1037
1038 static void
1039 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1040 {
1041         struct gicv3_its_softc *sc;
1042         struct gicv3_its_irqsrc *girq;
1043         uint8_t *conf;
1044
1045         sc = device_get_softc(dev);
1046         girq = (struct gicv3_its_irqsrc *)isrc;
1047         conf = sc->sc_conf_base;
1048
1049         conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1050
1051         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1052                 /* Clean D-cache under command. */
1053                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1054         } else {
1055                 /* DSB inner shareable, store */
1056                 dsb(ishst);
1057         }
1058
1059         its_cmd_inv(dev, girq->gi_its_dev, girq);
1060 }
1061
1062 static int
1063 gicv3_its_intr(void *arg, uintptr_t irq)
1064 {
1065         struct gicv3_its_softc *sc = arg;
1066         struct gicv3_its_irqsrc *girq;
1067         struct trapframe *tf;
1068
1069         irq -= sc->sc_irq_base;
1070         girq = sc->sc_irqs[irq];
1071         if (girq == NULL)
1072                 panic("gicv3_its_intr: Invalid interrupt %ld",
1073                     irq + sc->sc_irq_base);
1074
1075         tf = curthread->td_intr_frame;
1076         intr_isrc_dispatch(&girq->gi_isrc, tf);
1077         return (FILTER_HANDLED);
1078 }
1079
1080 static void
1081 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1082 {
1083         struct gicv3_its_irqsrc *girq;
1084
1085         girq = (struct gicv3_its_irqsrc *)isrc;
1086         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1087 }
1088
1089 static void
1090 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1091 {
1092
1093 }
1094
1095 static void
1096 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1097 {
1098         struct gicv3_its_irqsrc *girq;
1099
1100         girq = (struct gicv3_its_irqsrc *)isrc;
1101         gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1102 }
1103
1104 static int
1105 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1106 {
1107         struct gicv3_its_softc *sc;
1108
1109         sc = device_get_softc(dev);
1110         if (CPU_EMPTY(&isrc->isrc_cpu)) {
1111                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1112                     &sc->sc_cpus);
1113                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1114         }
1115
1116         return (0);
1117 }
1118
1119 static int
1120 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1121 {
1122         struct gicv3_its_irqsrc *girq;
1123
1124         gicv3_its_select_cpu(dev, isrc);
1125
1126         girq = (struct gicv3_its_irqsrc *)isrc;
1127         its_cmd_movi(dev, girq);
1128         return (0);
1129 }
1130
1131 static int
1132 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1133     struct intr_irqsrc **isrcp)
1134 {
1135
1136         /*
1137          * This should never happen, we only call this function to map
1138          * interrupts found before the controller driver is ready.
1139          */
1140         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1141 }
1142
1143 static int
1144 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1145     struct resource *res, struct intr_map_data *data)
1146 {
1147
1148         /* Bind the interrupt to a CPU */
1149         gicv3_its_bind_intr(dev, isrc);
1150
1151         return (0);
1152 }
1153
1154 #ifdef SMP
1155 static void
1156 gicv3_its_init_secondary(device_t dev)
1157 {
1158         struct gicv3_its_softc *sc;
1159
1160         sc = device_get_softc(dev);
1161
1162         /*
1163          * This is fatal as otherwise we may bind interrupts to this CPU.
1164          * We need a way to tell the interrupt framework to only bind to a
1165          * subset of given CPUs when it performs the shuffle.
1166          */
1167         if (its_init_cpu(dev, sc) != 0)
1168                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1169                     PCPU_GET(cpuid));
1170 }
1171 #endif
1172
1173 static uint32_t
1174 its_get_devid(device_t pci_dev)
1175 {
1176         uintptr_t id;
1177
1178         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1179                 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1180                     device_get_nameunit(pci_dev));
1181
1182         return (id);
1183 }
1184
1185 static struct its_dev *
1186 its_device_find(device_t dev, device_t child)
1187 {
1188         struct gicv3_its_softc *sc;
1189         struct its_dev *its_dev = NULL;
1190
1191         sc = device_get_softc(dev);
1192
1193         mtx_lock_spin(&sc->sc_its_dev_lock);
1194         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1195                 if (its_dev->pci_dev == child)
1196                         break;
1197         }
1198         mtx_unlock_spin(&sc->sc_its_dev_lock);
1199
1200         return (its_dev);
1201 }
1202
1203 static struct its_dev *
1204 its_device_get(device_t dev, device_t child, u_int nvecs)
1205 {
1206         struct gicv3_its_softc *sc;
1207         struct its_dev *its_dev;
1208         vmem_addr_t irq_base;
1209         size_t esize;
1210
1211         sc = device_get_softc(dev);
1212
1213         its_dev = its_device_find(dev, child);
1214         if (its_dev != NULL)
1215                 return (its_dev);
1216
1217         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1218         if (its_dev == NULL)
1219                 return (NULL);
1220
1221         its_dev->pci_dev = child;
1222         its_dev->devid = its_get_devid(child);
1223
1224         its_dev->lpis.lpi_busy = 0;
1225         its_dev->lpis.lpi_num = nvecs;
1226         its_dev->lpis.lpi_free = nvecs;
1227
1228         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1229             &irq_base) != 0) {
1230                 free(its_dev, M_GICV3_ITS);
1231                 return (NULL);
1232         }
1233         its_dev->lpis.lpi_base = irq_base;
1234
1235         /* Get ITT entry size */
1236         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1237
1238         /*
1239          * Allocate ITT for this device.
1240          * PA has to be 256 B aligned. At least two entries for device.
1241          */
1242         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1243         its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
1244             M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1245             LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1246         if (its_dev->itt == 0) {
1247                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1248                 free(its_dev, M_GICV3_ITS);
1249                 return (NULL);
1250         }
1251
1252         /* Make sure device sees zeroed ITT. */
1253         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1254                 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1255
1256         mtx_lock_spin(&sc->sc_its_dev_lock);
1257         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1258         mtx_unlock_spin(&sc->sc_its_dev_lock);
1259
1260         /* Map device to its ITT */
1261         its_cmd_mapd(dev, its_dev, 1);
1262
1263         return (its_dev);
1264 }
1265
1266 static void
1267 its_device_release(device_t dev, struct its_dev *its_dev)
1268 {
1269         struct gicv3_its_softc *sc;
1270
1271         KASSERT(its_dev->lpis.lpi_busy == 0,
1272             ("its_device_release: Trying to release an inuse ITS device"));
1273
1274         /* Unmap device in ITS */
1275         its_cmd_mapd(dev, its_dev, 0);
1276
1277         sc = device_get_softc(dev);
1278
1279         /* Remove the device from the list of devices */
1280         mtx_lock_spin(&sc->sc_its_dev_lock);
1281         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1282         mtx_unlock_spin(&sc->sc_its_dev_lock);
1283
1284         /* Free ITT */
1285         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1286         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1287
1288         /* Free the IRQ allocation */
1289         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1290             its_dev->lpis.lpi_num);
1291
1292         free(its_dev, M_GICV3_ITS);
1293 }
1294
1295 static struct gicv3_its_irqsrc *
1296 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1297 {
1298         struct gicv3_its_irqsrc *girq = NULL;
1299
1300         KASSERT(sc->sc_irqs[irq] == NULL,
1301             ("%s: Interrupt %u already allocated", __func__, irq));
1302         mtx_lock_spin(&sc->sc_its_dev_lock);
1303         if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1304                 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1305                 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1306         }
1307         mtx_unlock_spin(&sc->sc_its_dev_lock);
1308         if (girq == NULL) {
1309                 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1310                     M_NOWAIT | M_ZERO);
1311                 if (girq == NULL)
1312                         return (NULL);
1313                 girq->gi_id = -1;
1314                 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1315                     "%s,%u", device_get_nameunit(dev), irq) != 0) {
1316                         free(girq, M_GICV3_ITS);
1317                         return (NULL);
1318                 }
1319         }
1320         girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1321         sc->sc_irqs[irq] = girq;
1322
1323         return (girq);
1324 }
1325
1326 static void
1327 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1328     struct gicv3_its_irqsrc *girq)
1329 {
1330         u_int irq;
1331
1332         mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1333
1334         irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1335         sc->sc_irqs[irq] = NULL;
1336
1337         girq->gi_id = -1;
1338         girq->gi_its_dev = NULL;
1339         TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1340 }
1341
1342 static int
1343 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1344     device_t *pic, struct intr_irqsrc **srcs)
1345 {
1346         struct gicv3_its_softc *sc;
1347         struct gicv3_its_irqsrc *girq;
1348         struct its_dev *its_dev;
1349         u_int irq;
1350         int i;
1351
1352         its_dev = its_device_get(dev, child, count);
1353         if (its_dev == NULL)
1354                 return (ENXIO);
1355
1356         KASSERT(its_dev->lpis.lpi_free >= count,
1357             ("gicv3_its_alloc_msi: No free LPIs"));
1358         sc = device_get_softc(dev);
1359         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1360             its_dev->lpis.lpi_free;
1361
1362         /* Allocate the irqsrc for each MSI */
1363         for (i = 0; i < count; i++, irq++) {
1364                 its_dev->lpis.lpi_free--;
1365                 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1366                     sc, irq);
1367                 if (srcs[i] == NULL)
1368                         break;
1369         }
1370
1371         /* The allocation failed, release them */
1372         if (i != count) {
1373                 mtx_lock_spin(&sc->sc_its_dev_lock);
1374                 for (i = 0; i < count; i++) {
1375                         girq = (struct gicv3_its_irqsrc *)srcs[i];
1376                         if (girq == NULL)
1377                                 break;
1378                         gicv3_its_release_irqsrc(sc, girq);
1379                         srcs[i] = NULL;
1380                 }
1381                 mtx_unlock_spin(&sc->sc_its_dev_lock);
1382                 return (ENXIO);
1383         }
1384
1385         /* Finish the allocation now we have all MSI irqsrcs */
1386         for (i = 0; i < count; i++) {
1387                 girq = (struct gicv3_its_irqsrc *)srcs[i];
1388                 girq->gi_id = i;
1389                 girq->gi_its_dev = its_dev;
1390
1391                 /* Map the message to the given IRQ */
1392                 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1393                 its_cmd_mapti(dev, girq);
1394         }
1395         its_dev->lpis.lpi_busy += count;
1396         *pic = dev;
1397
1398         return (0);
1399 }
1400
1401 static int
1402 gicv3_its_release_msi(device_t dev, device_t child, int count,
1403     struct intr_irqsrc **isrc)
1404 {
1405         struct gicv3_its_softc *sc;
1406         struct gicv3_its_irqsrc *girq;
1407         struct its_dev *its_dev;
1408         int i;
1409
1410         its_dev = its_device_find(dev, child);
1411
1412         KASSERT(its_dev != NULL,
1413             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1414              "no ITS device"));
1415         KASSERT(its_dev->lpis.lpi_busy >= count,
1416             ("gicv3_its_release_msi: Releasing more interrupts than "
1417              "were allocated: releasing %d, allocated %d", count,
1418              its_dev->lpis.lpi_busy));
1419
1420         sc = device_get_softc(dev);
1421         mtx_lock_spin(&sc->sc_its_dev_lock);
1422         for (i = 0; i < count; i++) {
1423                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1424                 gicv3_its_release_irqsrc(sc, girq);
1425         }
1426         mtx_unlock_spin(&sc->sc_its_dev_lock);
1427         its_dev->lpis.lpi_busy -= count;
1428
1429         if (its_dev->lpis.lpi_busy == 0)
1430                 its_device_release(dev, its_dev);
1431
1432         return (0);
1433 }
1434
1435 static int
1436 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1437     struct intr_irqsrc **isrcp)
1438 {
1439         struct gicv3_its_softc *sc;
1440         struct gicv3_its_irqsrc *girq;
1441         struct its_dev *its_dev;
1442         u_int nvecs, irq;
1443
1444         nvecs = pci_msix_count(child);
1445         its_dev = its_device_get(dev, child, nvecs);
1446         if (its_dev == NULL)
1447                 return (ENXIO);
1448
1449         KASSERT(its_dev->lpis.lpi_free > 0,
1450             ("gicv3_its_alloc_msix: No free LPIs"));
1451         sc = device_get_softc(dev);
1452         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1453             its_dev->lpis.lpi_free;
1454
1455         girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1456         if (girq == NULL)
1457                 return (ENXIO);
1458         girq->gi_id = its_dev->lpis.lpi_busy;
1459         girq->gi_its_dev = its_dev;
1460
1461         its_dev->lpis.lpi_free--;
1462         its_dev->lpis.lpi_busy++;
1463
1464         /* Map the message to the given IRQ */
1465         gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1466         its_cmd_mapti(dev, girq);
1467
1468         *pic = dev;
1469         *isrcp = (struct intr_irqsrc *)girq;
1470
1471         return (0);
1472 }
1473
1474 static int
1475 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1476 {
1477         struct gicv3_its_softc *sc;
1478         struct gicv3_its_irqsrc *girq;
1479         struct its_dev *its_dev;
1480
1481         its_dev = its_device_find(dev, child);
1482
1483         KASSERT(its_dev != NULL,
1484             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1485              "no ITS device"));
1486         KASSERT(its_dev->lpis.lpi_busy > 0,
1487             ("gicv3_its_release_msix: Releasing more interrupts than "
1488              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1489
1490         sc = device_get_softc(dev);
1491         girq = (struct gicv3_its_irqsrc *)isrc;
1492         mtx_lock_spin(&sc->sc_its_dev_lock);
1493         gicv3_its_release_irqsrc(sc, girq);
1494         mtx_unlock_spin(&sc->sc_its_dev_lock);
1495         its_dev->lpis.lpi_busy--;
1496
1497         if (its_dev->lpis.lpi_busy == 0)
1498                 its_device_release(dev, its_dev);
1499
1500         return (0);
1501 }
1502
1503 static int
1504 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1505     uint64_t *addr, uint32_t *data)
1506 {
1507         struct gicv3_its_softc *sc;
1508         struct gicv3_its_irqsrc *girq;
1509
1510         sc = device_get_softc(dev);
1511         girq = (struct gicv3_its_irqsrc *)isrc;
1512
1513         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1514         *data = girq->gi_id;
1515
1516         return (0);
1517 }
1518
1519 #ifdef IOMMU
1520 static int
1521 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1522 {
1523         struct gicv3_its_softc *sc;
1524         struct iommu_ctx *ctx;
1525         int error;
1526
1527         sc = device_get_softc(dev);
1528         ctx = iommu_get_dev_ctx(child);
1529         if (ctx == NULL)
1530                 return (ENXIO);
1531         /* Map the page containing the GITS_TRANSLATER register. */
1532         error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1533             IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1534         *domain = iommu_get_ctx_domain(ctx);
1535
1536         return (error);
1537 }
1538
1539 static void
1540 gicv3_iommu_deinit(device_t dev, device_t child)
1541 {
1542         struct iommu_ctx *ctx;
1543
1544         ctx = iommu_get_dev_ctx(child);
1545         if (ctx == NULL)
1546                 return;
1547
1548         iommu_unmap_msi(ctx);
1549 }
1550 #endif
1551
1552 /*
1553  * Commands handling.
1554  */
1555
1556 static __inline void
1557 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1558 {
1559         /* Command field: DW0 [7:0] */
1560         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1561         cmd->cmd_dword[0] |= htole64(cmd_type);
1562 }
1563
1564 static __inline void
1565 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1566 {
1567         /* Device ID field: DW0 [63:32] */
1568         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1569         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1570 }
1571
1572 static __inline void
1573 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1574 {
1575         /* Size field: DW1 [4:0] */
1576         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1577         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1578 }
1579
1580 static __inline void
1581 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1582 {
1583         /* ID field: DW1 [31:0] */
1584         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1585         cmd->cmd_dword[1] |= htole64(id);
1586 }
1587
1588 static __inline void
1589 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1590 {
1591         /* Physical ID field: DW1 [63:32] */
1592         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1593         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1594 }
1595
1596 static __inline void
1597 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1598 {
1599         /* Collection field: DW2 [16:0] */
1600         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1601         cmd->cmd_dword[2] |= htole64(col_id);
1602 }
1603
1604 static __inline void
1605 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1606 {
1607         /* Target Address field: DW2 [47:16] */
1608         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1609         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1610 }
1611
1612 static __inline void
1613 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1614 {
1615         /* ITT Address field: DW2 [47:8] */
1616         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1617         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1618 }
1619
1620 static __inline void
1621 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1622 {
1623         /* Valid field: DW2 [63] */
1624         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1625         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1626 }
1627
1628 static inline bool
1629 its_cmd_queue_full(struct gicv3_its_softc *sc)
1630 {
1631         size_t read_idx, next_write_idx;
1632
1633         /* Get the index of the next command */
1634         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1635             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1636         /* And the index of the current command being read */
1637         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1638
1639         /*
1640          * The queue is full when the write offset points
1641          * at the command before the current read offset.
1642          */
1643         return (next_write_idx == read_idx);
1644 }
1645
1646 static inline void
1647 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1648 {
1649
1650         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1651                 /* Clean D-cache under command. */
1652                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1653         } else {
1654                 /* DSB inner shareable, store */
1655                 dsb(ishst);
1656         }
1657
1658 }
1659
1660 static inline uint64_t
1661 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1662 {
1663         uint64_t off;
1664
1665         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1666
1667         return (off);
1668 }
1669
1670 static void
1671 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1672     struct its_cmd *cmd_last)
1673 {
1674         struct gicv3_its_softc *sc;
1675         uint64_t first, last, read;
1676         size_t us_left;
1677
1678         sc = device_get_softc(dev);
1679
1680         /*
1681          * XXX ARM64TODO: This is obviously a significant delay.
1682          * The reason for that is that currently the time frames for
1683          * the command to complete are not known.
1684          */
1685         us_left = 1000000;
1686
1687         first = its_cmd_cwriter_offset(sc, cmd_first);
1688         last = its_cmd_cwriter_offset(sc, cmd_last);
1689
1690         for (;;) {
1691                 read = gic_its_read_8(sc, GITS_CREADR);
1692                 if (first < last) {
1693                         if (read < first || read >= last)
1694                                 break;
1695                 } else if (read < first && read >= last)
1696                         break;
1697
1698                 if (us_left-- == 0) {
1699                         /* This means timeout */
1700                         device_printf(dev,
1701                             "Timeout while waiting for CMD completion.\n");
1702                         return;
1703                 }
1704                 DELAY(1);
1705         }
1706 }
1707
1708 static struct its_cmd *
1709 its_cmd_alloc_locked(device_t dev)
1710 {
1711         struct gicv3_its_softc *sc;
1712         struct its_cmd *cmd;
1713         size_t us_left;
1714
1715         sc = device_get_softc(dev);
1716
1717         /*
1718          * XXX ARM64TODO: This is obviously a significant delay.
1719          * The reason for that is that currently the time frames for
1720          * the command to complete (and therefore free the descriptor)
1721          * are not known.
1722          */
1723         us_left = 1000000;
1724
1725         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1726         while (its_cmd_queue_full(sc)) {
1727                 if (us_left-- == 0) {
1728                         /* Timeout while waiting for free command */
1729                         device_printf(dev,
1730                             "Timeout while waiting for free command\n");
1731                         return (NULL);
1732                 }
1733                 DELAY(1);
1734         }
1735
1736         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1737         sc->sc_its_cmd_next_idx++;
1738         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1739
1740         return (cmd);
1741 }
1742
1743 static uint64_t
1744 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1745 {
1746         uint64_t target;
1747         uint8_t cmd_type;
1748         u_int size;
1749
1750         cmd_type = desc->cmd_type;
1751         target = ITS_TARGET_NONE;
1752
1753         switch (cmd_type) {
1754         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1755                 target = desc->cmd_desc_movi.col->col_target;
1756                 cmd_format_command(cmd, ITS_CMD_MOVI);
1757                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1758                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1759                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1760                 break;
1761         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1762                 target = desc->cmd_desc_sync.col->col_target;
1763                 cmd_format_command(cmd, ITS_CMD_SYNC);
1764                 cmd_format_target(cmd, target);
1765                 break;
1766         case ITS_CMD_MAPD:      /* Assign ITT to device */
1767                 cmd_format_command(cmd, ITS_CMD_MAPD);
1768                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1769                 /*
1770                  * Size describes number of bits to encode interrupt IDs
1771                  * supported by the device minus one.
1772                  * When V (valid) bit is zero, this field should be written
1773                  * as zero.
1774                  */
1775                 if (desc->cmd_desc_mapd.valid != 0) {
1776                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1777                         size = MAX(1, size) - 1;
1778                 } else
1779                         size = 0;
1780
1781                 cmd_format_size(cmd, size);
1782                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1783                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1784                 break;
1785         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1786                 target = desc->cmd_desc_mapc.col->col_target;
1787                 cmd_format_command(cmd, ITS_CMD_MAPC);
1788                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1789                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1790                 cmd_format_target(cmd, target);
1791                 break;
1792         case ITS_CMD_MAPTI:
1793                 target = desc->cmd_desc_mapvi.col->col_target;
1794                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1795                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1796                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1797                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1798                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1799                 break;
1800         case ITS_CMD_MAPI:
1801                 target = desc->cmd_desc_mapi.col->col_target;
1802                 cmd_format_command(cmd, ITS_CMD_MAPI);
1803                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1804                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1805                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1806                 break;
1807         case ITS_CMD_INV:
1808                 target = desc->cmd_desc_inv.col->col_target;
1809                 cmd_format_command(cmd, ITS_CMD_INV);
1810                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1811                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1812                 break;
1813         case ITS_CMD_INVALL:
1814                 cmd_format_command(cmd, ITS_CMD_INVALL);
1815                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1816                 break;
1817         default:
1818                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1819         }
1820
1821         return (target);
1822 }
1823
1824 static int
1825 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1826 {
1827         struct gicv3_its_softc *sc;
1828         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1829         struct its_col col_sync;
1830         struct its_cmd_desc desc_sync;
1831         uint64_t target, cwriter;
1832
1833         sc = device_get_softc(dev);
1834         mtx_lock_spin(&sc->sc_its_cmd_lock);
1835         cmd = its_cmd_alloc_locked(dev);
1836         if (cmd == NULL) {
1837                 device_printf(dev, "could not allocate ITS command\n");
1838                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1839                 return (EBUSY);
1840         }
1841
1842         target = its_cmd_prepare(cmd, desc);
1843         its_cmd_sync(sc, cmd);
1844
1845         if (target != ITS_TARGET_NONE) {
1846                 cmd_sync = its_cmd_alloc_locked(dev);
1847                 if (cmd_sync != NULL) {
1848                         desc_sync.cmd_type = ITS_CMD_SYNC;
1849                         col_sync.col_target = target;
1850                         desc_sync.cmd_desc_sync.col = &col_sync;
1851                         its_cmd_prepare(cmd_sync, &desc_sync);
1852                         its_cmd_sync(sc, cmd_sync);
1853                 }
1854         }
1855
1856         /* Update GITS_CWRITER */
1857         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
1858         gic_its_write_8(sc, GITS_CWRITER, cwriter);
1859         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1860         mtx_unlock_spin(&sc->sc_its_cmd_lock);
1861
1862         its_cmd_wait_completion(dev, cmd, cmd_write);
1863
1864         return (0);
1865 }
1866
1867 /* Handlers to send commands */
1868 static void
1869 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
1870 {
1871         struct gicv3_its_softc *sc;
1872         struct its_cmd_desc desc;
1873         struct its_col *col;
1874
1875         sc = device_get_softc(dev);
1876         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1877
1878         desc.cmd_type = ITS_CMD_MOVI;
1879         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
1880         desc.cmd_desc_movi.col = col;
1881         desc.cmd_desc_movi.id = girq->gi_id;
1882
1883         its_cmd_send(dev, &desc);
1884 }
1885
1886 static void
1887 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
1888 {
1889         struct its_cmd_desc desc;
1890
1891         desc.cmd_type = ITS_CMD_MAPC;
1892         desc.cmd_desc_mapc.col = col;
1893         /*
1894          * Valid bit set - map the collection.
1895          * Valid bit cleared - unmap the collection.
1896          */
1897         desc.cmd_desc_mapc.valid = valid;
1898
1899         its_cmd_send(dev, &desc);
1900 }
1901
1902 static void
1903 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
1904 {
1905         struct gicv3_its_softc *sc;
1906         struct its_cmd_desc desc;
1907         struct its_col *col;
1908         u_int col_id;
1909
1910         sc = device_get_softc(dev);
1911
1912         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
1913         col = sc->sc_its_cols[col_id];
1914
1915         desc.cmd_type = ITS_CMD_MAPTI;
1916         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
1917         desc.cmd_desc_mapvi.col = col;
1918         /* The EventID sent to the device */
1919         desc.cmd_desc_mapvi.id = girq->gi_id;
1920         /* The physical interrupt presented to softeware */
1921         desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
1922
1923         its_cmd_send(dev, &desc);
1924 }
1925
1926 static void
1927 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
1928 {
1929         struct its_cmd_desc desc;
1930
1931         desc.cmd_type = ITS_CMD_MAPD;
1932         desc.cmd_desc_mapd.its_dev = its_dev;
1933         desc.cmd_desc_mapd.valid = valid;
1934
1935         its_cmd_send(dev, &desc);
1936 }
1937
1938 static void
1939 its_cmd_inv(device_t dev, struct its_dev *its_dev,
1940     struct gicv3_its_irqsrc *girq)
1941 {
1942         struct gicv3_its_softc *sc;
1943         struct its_cmd_desc desc;
1944         struct its_col *col;
1945
1946         sc = device_get_softc(dev);
1947         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1948
1949         desc.cmd_type = ITS_CMD_INV;
1950         /* The EventID sent to the device */
1951         desc.cmd_desc_inv.pid = girq->gi_id;
1952         desc.cmd_desc_inv.its_dev = its_dev;
1953         desc.cmd_desc_inv.col = col;
1954
1955         its_cmd_send(dev, &desc);
1956 }
1957
1958 static void
1959 its_cmd_invall(device_t dev, struct its_col *col)
1960 {
1961         struct its_cmd_desc desc;
1962
1963         desc.cmd_type = ITS_CMD_INVALL;
1964         desc.cmd_desc_invall.col = col;
1965
1966         its_cmd_send(dev, &desc);
1967 }
1968
1969 #ifdef FDT
1970 static device_probe_t gicv3_its_fdt_probe;
1971 static device_attach_t gicv3_its_fdt_attach;
1972
1973 static device_method_t gicv3_its_fdt_methods[] = {
1974         /* Device interface */
1975         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
1976         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
1977
1978         /* End */
1979         DEVMETHOD_END
1980 };
1981
1982 #define its_baseclasses its_fdt_baseclasses
1983 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
1984     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1985 #undef its_baseclasses
1986
1987 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
1988     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1989
1990 static int
1991 gicv3_its_fdt_probe(device_t dev)
1992 {
1993
1994         if (!ofw_bus_status_okay(dev))
1995                 return (ENXIO);
1996
1997         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
1998                 return (ENXIO);
1999
2000         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2001         return (BUS_PROBE_DEFAULT);
2002 }
2003
2004 static int
2005 gicv3_its_fdt_attach(device_t dev)
2006 {
2007         struct gicv3_its_softc *sc;
2008         phandle_t xref;
2009         int err;
2010
2011         sc = device_get_softc(dev);
2012         sc->dev = dev;
2013         err = gicv3_its_attach(dev);
2014         if (err != 0)
2015                 return (err);
2016
2017         /* Register this device as a interrupt controller */
2018         xref = OF_xref_from_node(ofw_bus_get_node(dev));
2019         sc->sc_pic = intr_pic_register(dev, xref);
2020         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2021             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2022         if (err != 0) {
2023                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2024                 return (err);
2025         }
2026
2027         /* Register this device to handle MSI interrupts */
2028         err = intr_msi_register(dev, xref);
2029         if (err != 0) {
2030                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2031                 return (err);
2032         }
2033
2034         return (0);
2035 }
2036 #endif
2037
2038 #ifdef DEV_ACPI
2039 static device_probe_t gicv3_its_acpi_probe;
2040 static device_attach_t gicv3_its_acpi_attach;
2041
2042 static device_method_t gicv3_its_acpi_methods[] = {
2043         /* Device interface */
2044         DEVMETHOD(device_probe,         gicv3_its_acpi_probe),
2045         DEVMETHOD(device_attach,        gicv3_its_acpi_attach),
2046
2047         /* End */
2048         DEVMETHOD_END
2049 };
2050
2051 #define its_baseclasses its_acpi_baseclasses
2052 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2053     sizeof(struct gicv3_its_softc), gicv3_its_driver);
2054 #undef its_baseclasses
2055
2056 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2057     BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2058
2059 static int
2060 gicv3_its_acpi_probe(device_t dev)
2061 {
2062
2063         if (gic_get_bus(dev) != GIC_BUS_ACPI)
2064                 return (EINVAL);
2065
2066         if (gic_get_hw_rev(dev) < 3)
2067                 return (EINVAL);
2068
2069         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2070         return (BUS_PROBE_DEFAULT);
2071 }
2072
2073 static int
2074 gicv3_its_acpi_attach(device_t dev)
2075 {
2076         struct gicv3_its_softc *sc;
2077         struct gic_v3_devinfo *di;
2078         int err;
2079
2080         sc = device_get_softc(dev);
2081         sc->dev = dev;
2082         err = gicv3_its_attach(dev);
2083         if (err != 0)
2084                 return (err);
2085
2086         di = device_get_ivars(dev);
2087         sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2088         err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2089             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2090         if (err != 0) {
2091                 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2092                 return (err);
2093         }
2094
2095         /* Register this device to handle MSI interrupts */
2096         err = intr_msi_register(dev, di->msi_xref);
2097         if (err != 0) {
2098                 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2099                 return (err);
2100         }
2101
2102         return (0);
2103 }
2104 #endif