sys/arm64/arm64/gicv3_its.c

   1 /*-
   2  * Copyright (c) 2015-2016 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Andrew Turner under
   6  * the sponsorship of the FreeBSD Foundation.
   7  *
   8  * This software was developed by Semihalf under
   9  * the sponsorship of the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include "opt_platform.h"
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/bus.h>
  41 #include <sys/cpuset.h>
  42 #include <sys/endian.h>
  43 #include <sys/kernel.h>
  44 #include <sys/malloc.h>
  45 #include <sys/module.h>
  46 #include <sys/proc.h>
  47 #include <sys/queue.h>
  48 #include <sys/rman.h>
  49 #include <sys/smp.h>
  50 #include <sys/vmem.h>
  51
  52 #include <vm/vm.h>
  53 #include <vm/pmap.h>
  54
  55 #include <machine/bus.h>
  56 #include <machine/intr.h>
  57
  58 #include <arm/arm/gic_common.h>
  59 #include <arm64/arm64/gic_v3_reg.h>
  60 #include <arm64/arm64/gic_v3_var.h>
  61
  62 #ifdef FDT
  63 #include <dev/ofw/openfirm.h>
  64 #include <dev/ofw/ofw_bus.h>
  65 #include <dev/ofw/ofw_bus_subr.h>
  66 #endif
  67 #include <dev/pci/pcireg.h>
  68 #include <dev/pci/pcivar.h>
  69
  70 #include "pcib_if.h"
  71 #include "pic_if.h"
  72 #include "msi_if.h"
  73
  74 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
  75     "ARM GICv3 Interrupt Translation Service");
  76
  77 #define LPI_NIRQS               (64 * 1024)
  78
  79 /* The size and alignment of the command circular buffer */
  80 #define ITS_CMDQ_SIZE           (64 * 1024)     /* Must be a multiple of 4K */
  81 #define ITS_CMDQ_ALIGN          (64 * 1024)
  82
  83 #define LPI_CONFTAB_SIZE        LPI_NIRQS
  84 #define LPI_CONFTAB_ALIGN       (64 * 1024)
  85 #define LPI_CONFTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  86
  87 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
  88 #define LPI_PENDTAB_SIZE        ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
  89 #define LPI_PENDTAB_ALIGN       (64 * 1024)
  90 #define LPI_PENDTAB_MAX_ADDR    ((1ul << 48) - 1) /* We need a 47 bit PA */
  91
  92 #define LPI_INT_TRANS_TAB_ALIGN 256
  93 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
  94
  95 /* ITS commands encoding */
  96 #define ITS_CMD_MOVI            (0x01)
  97 #define ITS_CMD_SYNC            (0x05)
  98 #define ITS_CMD_MAPD            (0x08)
  99 #define ITS_CMD_MAPC            (0x09)
 100 #define ITS_CMD_MAPTI           (0x0a)
 101 #define ITS_CMD_MAPI            (0x0b)
 102 #define ITS_CMD_INV             (0x0c)
 103 #define ITS_CMD_INVALL          (0x0d)
 104 /* Command */
 105 #define CMD_COMMAND_MASK        (0xFFUL)
 106 /* PCI device ID */
 107 #define CMD_DEVID_SHIFT         (32)
 108 #define CMD_DEVID_MASK          (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
 109 /* Size of IRQ ID bitfield */
 110 #define CMD_SIZE_MASK           (0xFFUL)
 111 /* Virtual LPI ID */
 112 #define CMD_ID_MASK             (0xFFFFFFFFUL)
 113 /* Physical LPI ID */
 114 #define CMD_PID_SHIFT           (32)
 115 #define CMD_PID_MASK            (0xFFFFFFFFUL << CMD_PID_SHIFT)
 116 /* Collection */
 117 #define CMD_COL_MASK            (0xFFFFUL)
 118 /* Target (CPU or Re-Distributor) */
 119 #define CMD_TARGET_SHIFT        (16)
 120 #define CMD_TARGET_MASK         (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
 121 /* Interrupt Translation Table address */
 122 #define CMD_ITT_MASK            (0xFFFFFFFFFF00UL)
 123 /* Valid command bit */
 124 #define CMD_VALID_SHIFT         (63)
 125 #define CMD_VALID_MASK          (1UL << CMD_VALID_SHIFT)
 126
 127 #define ITS_TARGET_NONE         0xFBADBEEF
 128
 129 /* LPI chunk owned by ITS device */
 130 struct lpi_chunk {
 131         u_int   lpi_base;
 132         u_int   lpi_free;       /* First free LPI in set */
 133         u_int   lpi_num;        /* Total number of LPIs in chunk */
 134         u_int   lpi_busy;       /* Number of busy LPIs in chink */
 135 };
 136
 137 /* ITS device */
 138 struct its_dev {
 139         TAILQ_ENTRY(its_dev)    entry;
 140         /* PCI device */
 141         device_t                pci_dev;
 142         /* Device ID (i.e. PCI device ID) */
 143         uint32_t                devid;
 144         /* List of assigned LPIs */
 145         struct lpi_chunk        lpis;
 146         /* Virtual address of ITT */
 147         vm_offset_t             itt;
 148         size_t                  itt_size;
 149 };
 150
 151 /*
 152  * ITS command descriptor.
 153  * Idea for command description passing taken from Linux.
 154  */
 155 struct its_cmd_desc {
 156         uint8_t cmd_type;
 157
 158         union {
 159                 struct {
 160                         struct its_dev *its_dev;
 161                         struct its_col *col;
 162                         uint32_t id;
 163                 } cmd_desc_movi;
 164
 165                 struct {
 166                         struct its_col *col;
 167                 } cmd_desc_sync;
 168
 169                 struct {
 170                         struct its_col *col;
 171                         uint8_t valid;
 172                 } cmd_desc_mapc;
 173
 174                 struct {
 175                         struct its_dev *its_dev;
 176                         struct its_col *col;
 177                         uint32_t pid;
 178                         uint32_t id;
 179                 } cmd_desc_mapvi;
 180
 181                 struct {
 182                         struct its_dev *its_dev;
 183                         struct its_col *col;
 184                         uint32_t pid;
 185                 } cmd_desc_mapi;
 186
 187                 struct {
 188                         struct its_dev *its_dev;
 189                         uint8_t valid;
 190                 } cmd_desc_mapd;
 191
 192                 struct {
 193                         struct its_dev *its_dev;
 194                         struct its_col *col;
 195                         uint32_t pid;
 196                 } cmd_desc_inv;
 197
 198                 struct {
 199                         struct its_col *col;
 200                 } cmd_desc_invall;
 201         };
 202 };
 203
 204 /* ITS command. Each command is 32 bytes long */
 205 struct its_cmd {
 206         uint64_t        cmd_dword[4];   /* ITS command double word */
 207 };
 208
 209 /* An ITS private table */
 210 struct its_ptable {
 211         vm_offset_t     ptab_vaddr;
 212         unsigned long   ptab_size;
 213 };
 214
 215 /* ITS collection description. */
 216 struct its_col {
 217         uint64_t        col_target;     /* Target Re-Distributor */
 218         uint64_t        col_id;         /* Collection ID */
 219 };
 220
 221 struct gicv3_its_irqsrc {
 222         struct intr_irqsrc      gi_isrc;
 223         u_int                   gi_irq;
 224         struct its_dev          *gi_its_dev;
 225 };
 226
 227 struct gicv3_its_softc {
 228         struct intr_pic *sc_pic;
 229         struct resource *sc_its_res;
 230
 231         cpuset_t        sc_cpus;
 232         u_int           gic_irq_cpu;
 233
 234         struct its_ptable sc_its_ptab[GITS_BASER_NUM];
 235         struct its_col *sc_its_cols[MAXCPU];    /* Per-CPU collections */
 236
 237         /*
 238          * TODO: We should get these from the parent as we only want a
 239          * single copy of each across the interrupt controller.
 240          */
 241         vm_offset_t sc_conf_base;
 242         vm_offset_t sc_pend_base[MAXCPU];
 243
 244         /* Command handling */
 245         struct mtx sc_its_cmd_lock;
 246         struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
 247         size_t sc_its_cmd_next_idx;
 248
 249         vmem_t *sc_irq_alloc;
 250         struct gicv3_its_irqsrc *sc_irqs;
 251         u_int   sc_irq_base;
 252         u_int   sc_irq_length;
 253
 254         struct mtx sc_its_dev_lock;
 255         TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
 256
 257 #define ITS_FLAGS_CMDQ_FLUSH            0x00000001
 258 #define ITS_FLAGS_LPI_CONF_FLUSH        0x00000002
 259 #define ITS_FLAGS_ERRATA_CAVIUM_22375   0x00000004
 260         u_int sc_its_flags;
 261 };
 262
 263 typedef void (its_quirk_func_t)(device_t);
 264 static its_quirk_func_t its_quirk_cavium_22375;
 265
 266 static const struct {
 267         const char *desc;
 268         uint32_t iidr;
 269         uint32_t iidr_mask;
 270         its_quirk_func_t *func;
 271 } its_quirks[] = {
 272         {
 273                 /* Cavium ThunderX Pass 1.x */
 274                 .desc = "Cavoum ThunderX errata: 22375, 24313",
 275                 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
 276                     GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
 277                 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
 278                 .func = its_quirk_cavium_22375,
 279         },
 280 };
 281
 282 #define gic_its_read_4(sc, reg)                 \
 283     bus_read_4((sc)->sc_its_res, (reg))
 284 #define gic_its_read_8(sc, reg)                 \
 285     bus_read_8((sc)->sc_its_res, (reg))
 286
 287 #define gic_its_write_4(sc, reg, val)           \
 288     bus_write_4((sc)->sc_its_res, (reg), (val))
 289 #define gic_its_write_8(sc, reg, val)           \
 290     bus_write_8((sc)->sc_its_res, (reg), (val))
 291
 292 static device_attach_t gicv3_its_attach;
 293 static device_detach_t gicv3_its_detach;
 294
 295 static pic_disable_intr_t gicv3_its_disable_intr;
 296 static pic_enable_intr_t gicv3_its_enable_intr;
 297 static pic_map_intr_t gicv3_its_map_intr;
 298 static pic_setup_intr_t gicv3_its_setup_intr;
 299 static pic_post_filter_t gicv3_its_post_filter;
 300 static pic_post_ithread_t gicv3_its_post_ithread;
 301 static pic_pre_ithread_t gicv3_its_pre_ithread;
 302 static pic_bind_intr_t gicv3_its_bind_intr;
 303 #ifdef SMP
 304 static pic_init_secondary_t gicv3_its_init_secondary;
 305 #endif
 306 static msi_alloc_msi_t gicv3_its_alloc_msi;
 307 static msi_release_msi_t gicv3_its_release_msi;
 308 static msi_alloc_msix_t gicv3_its_alloc_msix;
 309 static msi_release_msix_t gicv3_its_release_msix;
 310 static msi_map_msi_t gicv3_its_map_msi;
 311
 312 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
 313 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
 314 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
 315 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
 316 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
 317 static void its_cmd_invall(device_t, struct its_col *);
 318
 319 static device_method_t gicv3_its_methods[] = {
 320         /* Device interface */
 321         DEVMETHOD(device_detach,        gicv3_its_detach),
 322
 323         /* Interrupt controller interface */
 324         DEVMETHOD(pic_disable_intr,     gicv3_its_disable_intr),
 325         DEVMETHOD(pic_enable_intr,      gicv3_its_enable_intr),
 326         DEVMETHOD(pic_map_intr,         gicv3_its_map_intr),
 327         DEVMETHOD(pic_setup_intr,       gicv3_its_setup_intr),
 328         DEVMETHOD(pic_post_filter,      gicv3_its_post_filter),
 329         DEVMETHOD(pic_post_ithread,     gicv3_its_post_ithread),
 330         DEVMETHOD(pic_pre_ithread,      gicv3_its_pre_ithread),
 331 #ifdef SMP
 332         DEVMETHOD(pic_bind_intr,        gicv3_its_bind_intr),
 333         DEVMETHOD(pic_init_secondary,   gicv3_its_init_secondary),
 334 #endif
 335
 336         /* MSI/MSI-X */
 337         DEVMETHOD(msi_alloc_msi,        gicv3_its_alloc_msi),
 338         DEVMETHOD(msi_release_msi,      gicv3_its_release_msi),
 339         DEVMETHOD(msi_alloc_msix,       gicv3_its_alloc_msix),
 340         DEVMETHOD(msi_release_msix,     gicv3_its_release_msix),
 341         DEVMETHOD(msi_map_msi,          gicv3_its_map_msi),
 342
 343         /* End */
 344         DEVMETHOD_END
 345 };
 346
 347 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
 348     sizeof(struct gicv3_its_softc));
 349
 350 static void
 351 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
 352 {
 353         vm_paddr_t cmd_paddr;
 354         uint64_t reg, tmp;
 355
 356         /* Set up the command circular buffer */
 357         sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS,
 358             M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0);
 359         sc->sc_its_cmd_next_idx = 0;
 360
 361         cmd_paddr = vtophys(sc->sc_its_cmd_base);
 362
 363         /* Set the base of the command buffer */
 364         reg = GITS_CBASER_VALID |
 365             (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
 366             cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
 367             (ITS_CMDQ_SIZE / 4096 - 1);
 368         gic_its_write_8(sc, GITS_CBASER, reg);
 369
 370         /* Read back to check for fixed value fields */
 371         tmp = gic_its_read_8(sc, GITS_CBASER);
 372
 373         if ((tmp & GITS_CBASER_SHARE_MASK) !=
 374             (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
 375                 /* Check if the hardware reported non-shareable */
 376                 if ((tmp & GITS_CBASER_SHARE_MASK) ==
 377                     (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
 378                         /* If so remove the cache attribute */
 379                         reg &= ~GITS_CBASER_CACHE_MASK;
 380                         reg &= ~GITS_CBASER_SHARE_MASK;
 381                         /* Set to Non-cacheable, Non-shareable */
 382                         reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
 383                         reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
 384
 385                         gic_its_write_8(sc, GITS_CBASER, reg);
 386                 }
 387
 388                 /* The command queue has to be flushed after each command */
 389                 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
 390         }
 391
 392         /* Get the next command from the start of the buffer */
 393         gic_its_write_8(sc, GITS_CWRITER, 0x0);
 394 }
 395
 396 static int
 397 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
 398 {
 399         vm_offset_t table;
 400         vm_paddr_t paddr;
 401         uint64_t cache, reg, share, tmp, type;
 402         size_t esize, its_tbl_size, nidents, nitspages, npages;
 403         int i, page_size;
 404         int devbits;
 405
 406         if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
 407                 /*
 408                  * GITS_TYPER[17:13] of ThunderX reports that device IDs
 409                  * are to be 21 bits in length. The entry size of the ITS
 410                  * table can be read from GITS_BASERn[52:48] and on ThunderX
 411                  * is supposed to be 8 bytes in length (for device table).
 412                  * Finally the page size that is to be used by ITS to access
 413                  * this table will be set to 64KB.
 414                  *
 415                  * This gives 0x200000 entries of size 0x8 bytes covered by
 416                  * 256 pages each of which 64KB in size. The number of pages
 417                  * (minus 1) should then be written to GITS_BASERn[7:0]. In
 418                  * that case this value would be 0xFF but on ThunderX the
 419                  * maximum value that HW accepts is 0xFD.
 420                  *
 421                  * Set an arbitrary number of device ID bits to 20 in order
 422                  * to limit the number of entries in ITS device table to
 423                  * 0x100000 and the table size to 8MB.
 424                  */
 425                 devbits = 20;
 426                 cache = 0;
 427         } else {
 428                 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
 429                 cache = GITS_BASER_CACHE_WAWB;
 430         }
 431         share = GITS_BASER_SHARE_IS;
 432         page_size = PAGE_SIZE_64K;
 433
 434         for (i = 0; i < GITS_BASER_NUM; i++) {
 435                 reg = gic_its_read_8(sc, GITS_BASER(i));
 436                 /* The type of table */
 437                 type = GITS_BASER_TYPE(reg);
 438                 /* The table entry size */
 439                 esize = GITS_BASER_ESIZE(reg);
 440
 441                 switch(type) {
 442                 case GITS_BASER_TYPE_DEV:
 443                         nidents = (1 << devbits);
 444                         its_tbl_size = esize * nidents;
 445                         its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K);
 446                         break;
 447                 case GITS_BASER_TYPE_VP:
 448                 case GITS_BASER_TYPE_PP: /* Undocumented? */
 449                 case GITS_BASER_TYPE_IC:
 450                         its_tbl_size = page_size;
 451                         break;
 452                 default:
 453                         continue;
 454                 }
 455                 npages = howmany(its_tbl_size, PAGE_SIZE);
 456
 457                 /* Allocate the table */
 458                 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,
 459                     M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
 460                     PAGE_SIZE_64K, 0);
 461
 462                 sc->sc_its_ptab[i].ptab_vaddr = table;
 463                 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;
 464
 465                 paddr = vtophys(table);
 466
 467                 while (1) {
 468                         nitspages = howmany(its_tbl_size, page_size);
 469
 470                         /* Clear the fields we will be setting */
 471                         reg &= ~(GITS_BASER_VALID |
 472                             GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
 473                             GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK |
 474                             GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
 475                             GITS_BASER_SIZE_MASK);
 476                         /* Set the new values */
 477                         reg |= GITS_BASER_VALID |
 478                             (cache << GITS_BASER_CACHE_SHIFT) |
 479                             (type << GITS_BASER_TYPE_SHIFT) |
 480                             ((esize - 1) << GITS_BASER_ESIZE_SHIFT) |
 481                             paddr | (share << GITS_BASER_SHARE_SHIFT) |
 482                             (nitspages - 1);
 483
 484                         switch (page_size) {
 485                         case PAGE_SIZE:         /* 4KB */
 486                                 reg |=
 487                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 488                                 break;
 489                         case PAGE_SIZE_16K:     /* 16KB */
 490                                 reg |=
 491                                     GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
 492                                 break;
 493                         case PAGE_SIZE_64K:     /* 64KB */
 494                                 reg |=
 495                                     GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
 496                                 break;
 497                         }
 498
 499                         gic_its_write_8(sc, GITS_BASER(i), reg);
 500
 501                         /* Read back to check */
 502                         tmp = gic_its_read_8(sc, GITS_BASER(i));
 503
 504                         /* Do the snareability masks line up? */
 505                         if ((tmp & GITS_BASER_SHARE_MASK) !=
 506                             (reg & GITS_BASER_SHARE_MASK)) {
 507                                 share = (tmp & GITS_BASER_SHARE_MASK) >>
 508                                     GITS_BASER_SHARE_SHIFT;
 509                                 continue;
 510                         }
 511
 512                         if ((tmp & GITS_BASER_PSZ_MASK) !=
 513                             (reg & GITS_BASER_PSZ_MASK)) {
 514                                 switch (page_size) {
 515                                 case PAGE_SIZE_16K:
 516                                         page_size = PAGE_SIZE;
 517                                         continue;
 518                                 case PAGE_SIZE_64K:
 519                                         page_size = PAGE_SIZE_16K;
 520                                         continue;
 521                                 }
 522                         }
 523
 524                         if (tmp != reg) {
 525                                 device_printf(dev, "GITS_BASER%d: "
 526                                     "unable to be updated: %lx != %lx\n",
 527                                     i, reg, tmp);
 528                                 return (ENXIO);
 529                         }
 530
 531                         /* We should have made all needed changes */
 532                         break;
 533                 }
 534         }
 535
 536         return (0);
 537 }
 538
 539 static void
 540 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
 541 {
 542
 543         sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE,
 544             M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN,
 545             0);
 546
 547         /* Set the default configuration */
 548         memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
 549             LPI_CONFTAB_SIZE);
 550
 551         /* Flush the table to memory */
 552         cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE);
 553 }
 554
 555 static void
 556 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
 557 {
 558         int i;
 559
 560         for (i = 0; i <= mp_maxid; i++) {
 561                 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
 562                         continue;
 563
 564                 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
 565                     LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
 566                     0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
 567
 568                 /* Flush so the ITS can see the memory */
 569                 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base,
 570                     LPI_PENDTAB_SIZE);
 571         }
 572 }
 573
 574 static int
 575 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
 576 {
 577         device_t gicv3;
 578         vm_paddr_t target;
 579         uint64_t xbaser, tmp;
 580         uint32_t ctlr;
 581         u_int cpuid;
 582         int domain;
 583
 584         if (!CPU_ISSET(PCPU_GET(cpuid), &sc->sc_cpus))
 585                 return (0);
 586
 587         if (bus_get_domain(dev, &domain) == 0) {
 588                 if (PCPU_GET(domain) != domain)
 589                         return (0);
 590         }
 591
 592         gicv3 = device_get_parent(dev);
 593         cpuid = PCPU_GET(cpuid);
 594
 595         /* Check if the ITS is enabled on this CPU */
 596         if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) {
 597                 return (ENXIO);
 598         }
 599
 600         /* Disable LPIs */
 601         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 602         ctlr &= ~GICR_CTLR_LPI_ENABLE;
 603         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 604
 605         /* Make sure changes are observable my the GIC */
 606         dsb(sy);
 607
 608         /*
 609          * Set the redistributor base
 610          */
 611         xbaser = vtophys(sc->sc_conf_base) |
 612             (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
 613             (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
 614             (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
 615         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 616
 617         /* Check the cache attributes we set */
 618         tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
 619
 620         if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
 621             (xbaser & GICR_PROPBASER_SHARE_MASK)) {
 622                 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
 623                     (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
 624                         /* We need to mark as non-cacheable */
 625                         xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
 626                             GICR_PROPBASER_CACHE_MASK);
 627                         /* Non-cacheable */
 628                         xbaser |= GICR_PROPBASER_CACHE_NIN <<
 629                             GICR_PROPBASER_CACHE_SHIFT;
 630                         /* Non-sareable */
 631                         xbaser |= GICR_PROPBASER_SHARE_NS <<
 632                             GICR_PROPBASER_SHARE_SHIFT;
 633                         gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
 634                 }
 635                 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
 636         }
 637
 638         /*
 639          * Set the LPI pending table base
 640          */
 641         xbaser = vtophys(sc->sc_pend_base[cpuid]) |
 642             (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
 643             (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
 644
 645         gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
 646
 647         tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
 648
 649         if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
 650             (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
 651                 /* Clear the cahce and shareability bits */
 652                 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
 653                     GICR_PENDBASER_SHARE_MASK);
 654                 /* Mark as non-shareable */
 655                 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
 656                 /* And non-cacheable */
 657                 xbaser |= GICR_PENDBASER_CACHE_NIN <<
 658                     GICR_PENDBASER_CACHE_SHIFT;
 659         }
 660
 661         /* Enable LPIs */
 662         ctlr = gic_r_read_4(gicv3, GICR_CTLR);
 663         ctlr |= GICR_CTLR_LPI_ENABLE;
 664         gic_r_write_4(gicv3, GICR_CTLR, ctlr);
 665
 666         /* Make sure the GIC has seen everything */
 667         dsb(sy);
 668
 669         if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
 670                 /* This ITS wants the redistributor physical address */
 671                 target = vtophys(gicv3_get_redist_vaddr(dev));
 672         } else {
 673                 /* This ITS wants the unique processor number */
 674                 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER));
 675         }
 676
 677         sc->sc_its_cols[cpuid]->col_target = target;
 678         sc->sc_its_cols[cpuid]->col_id = cpuid;
 679
 680         its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
 681         its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
 682
 683         return (0);
 684 }
 685
 686 static int
 687 gicv3_its_attach(device_t dev)
 688 {
 689         struct gicv3_its_softc *sc;
 690         const char *name;
 691         uint32_t iidr;
 692         int domain, err, i, rid;
 693
 694         sc = device_get_softc(dev);
 695
 696         rid = 0;
 697         sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 698             RF_ACTIVE);
 699         if (sc->sc_its_res == NULL) {
 700                 device_printf(dev, "Could not allocate memory\n");
 701                 return (ENXIO);
 702         }
 703
 704         iidr = gic_its_read_4(sc, GITS_IIDR);
 705         for (i = 0; i < nitems(its_quirks); i++) {
 706                 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
 707                         if (bootverbose) {
 708                                 device_printf(dev, "Applying %s\n",
 709                                     its_quirks[i].desc);
 710                         }
 711                         its_quirks[i].func(dev);
 712                         break;
 713                 }
 714         }
 715
 716         /* Allocate the private tables */
 717         err = gicv3_its_table_init(dev, sc);
 718         if (err != 0)
 719                 return (err);
 720
 721         /* Protects access to the device list */
 722         mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
 723
 724         /* Protects access to the ITS command circular buffer. */
 725         mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
 726
 727         if (bus_get_domain(dev, &domain) == 0) {
 728                 CPU_ZERO(&sc->sc_cpus);
 729                 if (domain < MAXMEMDOM)
 730                         CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
 731         } else {
 732                 CPU_COPY(&all_cpus, &sc->sc_cpus);
 733         }
 734
 735         /* Allocate the command circular buffer */
 736         gicv3_its_cmdq_init(sc);
 737
 738         /* Allocate the per-CPU collections */
 739         for (int cpu = 0; cpu <= mp_maxid; cpu++)
 740                 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
 741                         sc->sc_its_cols[cpu] = malloc(
 742                             sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
 743                             M_WAITOK | M_ZERO);
 744                 else
 745                         sc->sc_its_cols[cpu] = NULL;
 746
 747         /* Enable the ITS */
 748         gic_its_write_4(sc, GITS_CTLR,
 749             gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN);
 750
 751         /* Create the LPI configuration table */
 752         gicv3_its_conftable_init(sc);
 753
 754         /* And the pending tebles */
 755         gicv3_its_pendtables_init(sc);
 756
 757         /* Enable LPIs on this CPU */
 758         its_init_cpu(dev, sc);
 759
 760         TAILQ_INIT(&sc->sc_its_dev_list);
 761
 762         /*
 763          * Create the vmem object to allocate INTRNG IRQs from. We try to
 764          * use all IRQs not already used by the GICv3.
 765          * XXX: This assumes there are no other interrupt controllers in the
 766          * system.
 767          */
 768         sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0,
 769             gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK);
 770
 771         sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
 772             M_GICV3_ITS, M_WAITOK | M_ZERO);
 773         name = device_get_nameunit(dev);
 774         for (i = 0; i < sc->sc_irq_length; i++) {
 775                 sc->sc_irqs[i].gi_irq = i;
 776                 err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0,
 777                     "%s,%u", name, i);
 778         }
 779
 780         return (0);
 781 }
 782
 783 static int
 784 gicv3_its_detach(device_t dev)
 785 {
 786
 787         return (ENXIO);
 788 }
 789
 790 static void
 791 its_quirk_cavium_22375(device_t dev)
 792 {
 793         struct gicv3_its_softc *sc;
 794
 795         sc = device_get_softc(dev);
 796         sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
 797 }
 798
 799 static void
 800 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
 801 {
 802         struct gicv3_its_softc *sc;
 803         struct gicv3_its_irqsrc *girq;
 804         uint8_t *conf;
 805
 806         sc = device_get_softc(dev);
 807         girq = (struct gicv3_its_irqsrc *)isrc;
 808         conf = (uint8_t *)sc->sc_conf_base;
 809
 810         conf[girq->gi_irq] &= ~LPI_CONF_ENABLE;
 811
 812         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
 813                 /* Clean D-cache under command. */
 814                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1);
 815         } else {
 816                 /* DSB inner shareable, store */
 817                 dsb(ishst);
 818         }
 819
 820         its_cmd_inv(dev, girq->gi_its_dev, girq);
 821 }
 822
 823 static void
 824 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
 825 {
 826         struct gicv3_its_softc *sc;
 827         struct gicv3_its_irqsrc *girq;
 828         uint8_t *conf;
 829
 830         sc = device_get_softc(dev);
 831         girq = (struct gicv3_its_irqsrc *)isrc;
 832         conf = (uint8_t *)sc->sc_conf_base;
 833
 834         conf[girq->gi_irq] |= LPI_CONF_ENABLE;
 835
 836         if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
 837                 /* Clean D-cache under command. */
 838                 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1);
 839         } else {
 840                 /* DSB inner shareable, store */
 841                 dsb(ishst);
 842         }
 843
 844         its_cmd_inv(dev, girq->gi_its_dev, girq);
 845 }
 846
 847 static int
 848 gicv3_its_intr(void *arg, uintptr_t irq)
 849 {
 850         struct gicv3_its_softc *sc = arg;
 851         struct gicv3_its_irqsrc *girq;
 852         struct trapframe *tf;
 853
 854         irq -= sc->sc_irq_base;
 855         girq = &sc->sc_irqs[irq];
 856         if (girq == NULL)
 857                 panic("gicv3_its_intr: Invalid interrupt %ld",
 858                     irq + sc->sc_irq_base);
 859
 860         tf = curthread->td_intr_frame;
 861         intr_isrc_dispatch(&girq->gi_isrc, tf);
 862         return (FILTER_HANDLED);
 863 }
 864
 865 static void
 866 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
 867 {
 868         struct gicv3_its_irqsrc *girq;
 869         struct gicv3_its_softc *sc;
 870
 871         sc = device_get_softc(dev);
 872         girq = (struct gicv3_its_irqsrc *)isrc;
 873         gicv3_its_disable_intr(dev, isrc);
 874         gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base);
 875 }
 876
 877 static void
 878 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
 879 {
 880
 881         gicv3_its_enable_intr(dev, isrc);
 882 }
 883
 884 static void
 885 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
 886 {
 887         struct gicv3_its_irqsrc *girq;
 888         struct gicv3_its_softc *sc;
 889
 890         sc = device_get_softc(dev);
 891         girq = (struct gicv3_its_irqsrc *)isrc;
 892         gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base);
 893 }
 894
 895 static int
 896 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
 897 {
 898         struct gicv3_its_irqsrc *girq;
 899         struct gicv3_its_softc *sc;
 900
 901         sc = device_get_softc(dev);
 902         girq = (struct gicv3_its_irqsrc *)isrc;
 903         if (CPU_EMPTY(&isrc->isrc_cpu)) {
 904                 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
 905                     &sc->sc_cpus);
 906                 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
 907         }
 908
 909         its_cmd_movi(dev, girq);
 910
 911         return (0);
 912 }
 913
 914 static int
 915 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
 916     struct intr_irqsrc **isrcp)
 917 {
 918
 919         /*
 920          * This should never happen, we only call this function to map
 921          * interrupts found before the controller driver is ready.
 922          */
 923         panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
 924 }
 925
 926 static int
 927 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
 928     struct resource *res, struct intr_map_data *data)
 929 {
 930
 931         /* Bind the interrupt to a CPU */
 932         gicv3_its_bind_intr(dev, isrc);
 933
 934         return (0);
 935 }
 936
 937 #ifdef SMP
 938 static void
 939 gicv3_its_init_secondary(device_t dev)
 940 {
 941         struct gicv3_its_softc *sc;
 942
 943         sc = device_get_softc(dev);
 944
 945         /*
 946          * This is fatal as otherwise we may bind interrupts to this CPU.
 947          * We need a way to tell the interrupt framework to only bind to a
 948          * subset of given CPUs when it performs the shuffle.
 949          */
 950         if (its_init_cpu(dev, sc) != 0)
 951                 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
 952                     PCPU_GET(cpuid));
 953 }
 954 #endif
 955
 956 static uint32_t
 957 its_get_devid(device_t pci_dev)
 958 {
 959         uintptr_t id;
 960
 961         if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
 962                 panic("its_get_devid: Unable to get the MSI DeviceID");
 963
 964         return (id);
 965 }
 966
 967 static struct its_dev *
 968 its_device_find(device_t dev, device_t child)
 969 {
 970         struct gicv3_its_softc *sc;
 971         struct its_dev *its_dev = NULL;
 972
 973         sc = device_get_softc(dev);
 974
 975         mtx_lock_spin(&sc->sc_its_dev_lock);
 976         TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
 977                 if (its_dev->pci_dev == child)
 978                         break;
 979         }
 980         mtx_unlock_spin(&sc->sc_its_dev_lock);
 981
 982         return (its_dev);
 983 }
 984
 985 static struct its_dev *
 986 its_device_get(device_t dev, device_t child, u_int nvecs)
 987 {
 988         struct gicv3_its_softc *sc;
 989         struct its_dev *its_dev;
 990         vmem_addr_t irq_base;
 991         size_t esize;
 992
 993         sc = device_get_softc(dev);
 994
 995         its_dev = its_device_find(dev, child);
 996         if (its_dev != NULL)
 997                 return (its_dev);
 998
 999         its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1000         if (its_dev == NULL)
1001                 return (NULL);
1002
1003         its_dev->pci_dev = child;
1004         its_dev->devid = its_get_devid(child);
1005
1006         its_dev->lpis.lpi_busy = 0;
1007         its_dev->lpis.lpi_num = nvecs;
1008         its_dev->lpis.lpi_free = nvecs;
1009
1010         if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1011             &irq_base) != 0) {
1012                 free(its_dev, M_GICV3_ITS);
1013                 return (NULL);
1014         }
1015         its_dev->lpis.lpi_base = irq_base;
1016
1017         /* Get ITT entry size */
1018         esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1019
1020         /*
1021          * Allocate ITT for this device.
1022          * PA has to be 256 B aligned. At least two entries for device.
1023          */
1024         its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1025         its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size,
1026             M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR,
1027             LPI_INT_TRANS_TAB_ALIGN, 0);
1028         if (its_dev->itt == 0) {
1029                 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1030                 free(its_dev, M_GICV3_ITS);
1031                 return (NULL);
1032         }
1033
1034         mtx_lock_spin(&sc->sc_its_dev_lock);
1035         TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1036         mtx_unlock_spin(&sc->sc_its_dev_lock);
1037
1038         /* Map device to its ITT */
1039         its_cmd_mapd(dev, its_dev, 1);
1040
1041         return (its_dev);
1042 }
1043
1044 static void
1045 its_device_release(device_t dev, struct its_dev *its_dev)
1046 {
1047         struct gicv3_its_softc *sc;
1048
1049         KASSERT(its_dev->lpis.lpi_busy == 0,
1050             ("its_device_release: Trying to release an inuse ITS device"));
1051
1052         /* Unmap device in ITS */
1053         its_cmd_mapd(dev, its_dev, 0);
1054
1055         sc = device_get_softc(dev);
1056
1057         /* Remove the device from the list of devices */
1058         mtx_lock_spin(&sc->sc_its_dev_lock);
1059         TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1060         mtx_unlock_spin(&sc->sc_its_dev_lock);
1061
1062         /* Free ITT */
1063         KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1064         contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1065
1066         /* Free the IRQ allocation */
1067         vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1068             its_dev->lpis.lpi_num);
1069
1070         free(its_dev, M_GICV3_ITS);
1071 }
1072
1073 static int
1074 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1075     device_t *pic, struct intr_irqsrc **srcs)
1076 {
1077         struct gicv3_its_softc *sc;
1078         struct gicv3_its_irqsrc *girq;
1079         struct its_dev *its_dev;
1080         u_int irq;
1081         int i;
1082
1083         its_dev = its_device_get(dev, child, count);
1084         if (its_dev == NULL)
1085                 return (ENXIO);
1086
1087         KASSERT(its_dev->lpis.lpi_free >= count,
1088             ("gicv3_its_alloc_msi: No free LPIs"));
1089         sc = device_get_softc(dev);
1090         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1091             its_dev->lpis.lpi_free;
1092         for (i = 0; i < count; i++, irq++) {
1093                 its_dev->lpis.lpi_free--;
1094                 girq = &sc->sc_irqs[irq];
1095                 girq->gi_its_dev = its_dev;
1096                 srcs[i] = (struct intr_irqsrc *)girq;
1097         }
1098         its_dev->lpis.lpi_busy += count;
1099         *pic = dev;
1100
1101         return (0);
1102 }
1103
1104 static int
1105 gicv3_its_release_msi(device_t dev, device_t child, int count,
1106     struct intr_irqsrc **isrc)
1107 {
1108         struct gicv3_its_softc *sc;
1109         struct gicv3_its_irqsrc *girq;
1110         struct its_dev *its_dev;
1111         int i;
1112
1113         sc = device_get_softc(dev);
1114         its_dev = its_device_find(dev, child);
1115
1116         KASSERT(its_dev != NULL,
1117             ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1118              "no ITS device"));
1119         KASSERT(its_dev->lpis.lpi_busy >= count,
1120             ("gicv3_its_release_msi: Releasing more interrupts than "
1121              "were allocated: releasing %d, allocated %d", count,
1122              its_dev->lpis.lpi_busy));
1123         for (i = 0; i < count; i++) {
1124                 girq = (struct gicv3_its_irqsrc *)isrc[i];
1125                 girq->gi_its_dev = NULL;
1126         }
1127         its_dev->lpis.lpi_busy -= count;
1128
1129         if (its_dev->lpis.lpi_busy == 0)
1130                 its_device_release(dev, its_dev);
1131
1132         return (0);
1133 }
1134
1135 static int
1136 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1137     struct intr_irqsrc **isrcp)
1138 {
1139         struct gicv3_its_softc *sc;
1140         struct gicv3_its_irqsrc *girq;
1141         struct its_dev *its_dev;
1142         u_int nvecs, irq;
1143
1144         nvecs = pci_msix_count(child);
1145         its_dev = its_device_get(dev, child, nvecs);
1146         if (its_dev == NULL)
1147                 return (ENXIO);
1148
1149         KASSERT(its_dev->lpis.lpi_free > 0,
1150             ("gicv3_its_alloc_msix: No free LPIs"));
1151         sc = device_get_softc(dev);
1152         irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1153             its_dev->lpis.lpi_free;
1154         its_dev->lpis.lpi_free--;
1155         its_dev->lpis.lpi_busy++;
1156         girq = &sc->sc_irqs[irq];
1157         girq->gi_its_dev = its_dev;
1158
1159         *pic = dev;
1160         *isrcp = (struct intr_irqsrc *)girq;
1161
1162         return (0);
1163 }
1164
1165 static int
1166 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1167 {
1168         struct gicv3_its_softc *sc;
1169         struct gicv3_its_irqsrc *girq;
1170         struct its_dev *its_dev;
1171
1172         sc = device_get_softc(dev);
1173         its_dev = its_device_find(dev, child);
1174
1175         KASSERT(its_dev != NULL,
1176             ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1177              "no ITS device"));
1178         KASSERT(its_dev->lpis.lpi_busy > 0,
1179             ("gicv3_its_release_msix: Releasing more interrupts than "
1180              "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1181         girq = (struct gicv3_its_irqsrc *)isrc;
1182         girq->gi_its_dev = NULL;
1183         its_dev->lpis.lpi_busy--;
1184
1185         if (its_dev->lpis.lpi_busy == 0)
1186                 its_device_release(dev, its_dev);
1187
1188         return (0);
1189 }
1190
1191 static int
1192 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1193     uint64_t *addr, uint32_t *data)
1194 {
1195         struct gicv3_its_softc *sc;
1196         struct gicv3_its_irqsrc *girq;
1197
1198         sc = device_get_softc(dev);
1199         girq = (struct gicv3_its_irqsrc *)isrc;
1200
1201         /* Map the message to the given IRQ */
1202         its_cmd_mapti(dev, girq);
1203
1204         *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1205         *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1206
1207         return (0);
1208 }
1209
1210 /*
1211  * Commands handling.
1212  */
1213
1214 static __inline void
1215 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1216 {
1217         /* Command field: DW0 [7:0] */
1218         cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1219         cmd->cmd_dword[0] |= htole64(cmd_type);
1220 }
1221
1222 static __inline void
1223 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1224 {
1225         /* Device ID field: DW0 [63:32] */
1226         cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1227         cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1228 }
1229
1230 static __inline void
1231 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1232 {
1233         /* Size field: DW1 [4:0] */
1234         cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1235         cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1236 }
1237
1238 static __inline void
1239 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1240 {
1241         /* ID field: DW1 [31:0] */
1242         cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1243         cmd->cmd_dword[1] |= htole64(id);
1244 }
1245
1246 static __inline void
1247 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1248 {
1249         /* Physical ID field: DW1 [63:32] */
1250         cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1251         cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1252 }
1253
1254 static __inline void
1255 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1256 {
1257         /* Collection field: DW2 [16:0] */
1258         cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1259         cmd->cmd_dword[2] |= htole64(col_id);
1260 }
1261
1262 static __inline void
1263 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1264 {
1265         /* Target Address field: DW2 [47:16] */
1266         cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1267         cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1268 }
1269
1270 static __inline void
1271 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1272 {
1273         /* ITT Address field: DW2 [47:8] */
1274         cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1275         cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1276 }
1277
1278 static __inline void
1279 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1280 {
1281         /* Valid field: DW2 [63] */
1282         cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1283         cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1284 }
1285
1286 static inline bool
1287 its_cmd_queue_full(struct gicv3_its_softc *sc)
1288 {
1289         size_t read_idx, next_write_idx;
1290
1291         /* Get the index of the next command */
1292         next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1293             (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1294         /* And the index of the current command being read */
1295         read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1296
1297         /*
1298          * The queue is full when the write offset points
1299          * at the command before the current read offset.
1300          */
1301         return (next_write_idx == read_idx);
1302 }
1303
1304 static inline void
1305 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1306 {
1307
1308         if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1309                 /* Clean D-cache under command. */
1310                 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1311         } else {
1312                 /* DSB inner shareable, store */
1313                 dsb(ishst);
1314         }
1315
1316 }
1317
1318 static inline uint64_t
1319 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1320 {
1321         uint64_t off;
1322
1323         off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1324
1325         return (off);
1326 }
1327
1328 static void
1329 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1330     struct its_cmd *cmd_last)
1331 {
1332         struct gicv3_its_softc *sc;
1333         uint64_t first, last, read;
1334         size_t us_left;
1335
1336         sc = device_get_softc(dev);
1337
1338         /*
1339          * XXX ARM64TODO: This is obviously a significant delay.
1340          * The reason for that is that currently the time frames for
1341          * the command to complete are not known.
1342          */
1343         us_left = 1000000;
1344
1345         first = its_cmd_cwriter_offset(sc, cmd_first);
1346         last = its_cmd_cwriter_offset(sc, cmd_last);
1347
1348         for (;;) {
1349                 read = gic_its_read_8(sc, GITS_CREADR);
1350                 if (first < last) {
1351                         if (read < first || read >= last)
1352                                 break;
1353                 } else if (read < first && read >= last)
1354                         break;
1355
1356                 if (us_left-- == 0) {
1357                         /* This means timeout */
1358                         device_printf(dev,
1359                             "Timeout while waiting for CMD completion.\n");
1360                         return;
1361                 }
1362                 DELAY(1);
1363         }
1364 }
1365
1366
1367 static struct its_cmd *
1368 its_cmd_alloc_locked(device_t dev)
1369 {
1370         struct gicv3_its_softc *sc;
1371         struct its_cmd *cmd;
1372         size_t us_left;
1373
1374         sc = device_get_softc(dev);
1375
1376         /*
1377          * XXX ARM64TODO: This is obviously a significant delay.
1378          * The reason for that is that currently the time frames for
1379          * the command to complete (and therefore free the descriptor)
1380          * are not known.
1381          */
1382         us_left = 1000000;
1383
1384         mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1385         while (its_cmd_queue_full(sc)) {
1386                 if (us_left-- == 0) {
1387                         /* Timeout while waiting for free command */
1388                         device_printf(dev,
1389                             "Timeout while waiting for free command\n");
1390                         return (NULL);
1391                 }
1392                 DELAY(1);
1393         }
1394
1395         cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1396         sc->sc_its_cmd_next_idx++;
1397         sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1398
1399         return (cmd);
1400 }
1401
1402 static uint64_t
1403 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1404 {
1405         uint64_t target;
1406         uint8_t cmd_type;
1407         u_int size;
1408         boolean_t error;
1409
1410         error = FALSE;
1411         cmd_type = desc->cmd_type;
1412         target = ITS_TARGET_NONE;
1413
1414         switch (cmd_type) {
1415         case ITS_CMD_MOVI:      /* Move interrupt ID to another collection */
1416                 target = desc->cmd_desc_movi.col->col_target;
1417                 cmd_format_command(cmd, ITS_CMD_MOVI);
1418                 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1419                 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1420                 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1421                 break;
1422         case ITS_CMD_SYNC:      /* Wait for previous commands completion */
1423                 target = desc->cmd_desc_sync.col->col_target;
1424                 cmd_format_command(cmd, ITS_CMD_SYNC);
1425                 cmd_format_target(cmd, target);
1426                 break;
1427         case ITS_CMD_MAPD:      /* Assign ITT to device */
1428                 cmd_format_command(cmd, ITS_CMD_MAPD);
1429                 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1430                 /*
1431                  * Size describes number of bits to encode interrupt IDs
1432                  * supported by the device minus one.
1433                  * When V (valid) bit is zero, this field should be written
1434                  * as zero.
1435                  */
1436                 if (desc->cmd_desc_mapd.valid != 0) {
1437                         size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1438                         size = MAX(1, size) - 1;
1439                 } else
1440                         size = 0;
1441
1442                 cmd_format_size(cmd, size);
1443                 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1444                 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1445                 break;
1446         case ITS_CMD_MAPC:      /* Map collection to Re-Distributor */
1447                 target = desc->cmd_desc_mapc.col->col_target;
1448                 cmd_format_command(cmd, ITS_CMD_MAPC);
1449                 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1450                 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1451                 cmd_format_target(cmd, target);
1452                 break;
1453         case ITS_CMD_MAPTI:
1454                 target = desc->cmd_desc_mapvi.col->col_target;
1455                 cmd_format_command(cmd, ITS_CMD_MAPTI);
1456                 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1457                 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1458                 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1459                 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
1460                 break;
1461         case ITS_CMD_MAPI:
1462                 target = desc->cmd_desc_mapi.col->col_target;
1463                 cmd_format_command(cmd, ITS_CMD_MAPI);
1464                 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1465                 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
1466                 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
1467                 break;
1468         case ITS_CMD_INV:
1469                 target = desc->cmd_desc_inv.col->col_target;
1470                 cmd_format_command(cmd, ITS_CMD_INV);
1471                 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1472                 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
1473                 break;
1474         case ITS_CMD_INVALL:
1475                 cmd_format_command(cmd, ITS_CMD_INVALL);
1476                 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1477                 break;
1478         default:
1479                 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
1480         }
1481
1482         return (target);
1483 }
1484
1485 static int
1486 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
1487 {
1488         struct gicv3_its_softc *sc;
1489         struct its_cmd *cmd, *cmd_sync, *cmd_write;
1490         struct its_col col_sync;
1491         struct its_cmd_desc desc_sync;
1492         uint64_t target, cwriter;
1493
1494         sc = device_get_softc(dev);
1495         mtx_lock_spin(&sc->sc_its_cmd_lock);
1496         cmd = its_cmd_alloc_locked(dev);
1497         if (cmd == NULL) {
1498                 device_printf(dev, "could not allocate ITS command\n");
1499                 mtx_unlock_spin(&sc->sc_its_cmd_lock);
1500                 return (EBUSY);
1501         }
1502
1503         target = its_cmd_prepare(cmd, desc);
1504         its_cmd_sync(sc, cmd);
1505
1506         if (target != ITS_TARGET_NONE) {
1507                 cmd_sync = its_cmd_alloc_locked(dev);
1508                 if (cmd_sync != NULL) {
1509                         desc_sync.cmd_type = ITS_CMD_SYNC;
1510                         col_sync.col_target = target;
1511                         desc_sync.cmd_desc_sync.col = &col_sync;
1512                         its_cmd_prepare(cmd_sync, &desc_sync);
1513                         its_cmd_sync(sc, cmd_sync);
1514                 }
1515         }
1516
1517         /* Update GITS_CWRITER */
1518         cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
1519         gic_its_write_8(sc, GITS_CWRITER, cwriter);
1520         cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1521         mtx_unlock_spin(&sc->sc_its_cmd_lock);
1522
1523         its_cmd_wait_completion(dev, cmd, cmd_write);
1524
1525         return (0);
1526 }
1527
1528 /* Handlers to send commands */
1529 static void
1530 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
1531 {
1532         struct gicv3_its_softc *sc;
1533         struct its_cmd_desc desc;
1534         struct its_col *col;
1535
1536         sc = device_get_softc(dev);
1537         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1538
1539         desc.cmd_type = ITS_CMD_MOVI;
1540         desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
1541         desc.cmd_desc_movi.col = col;
1542         desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1543
1544         its_cmd_send(dev, &desc);
1545 }
1546
1547 static void
1548 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
1549 {
1550         struct its_cmd_desc desc;
1551
1552         desc.cmd_type = ITS_CMD_MAPC;
1553         desc.cmd_desc_mapc.col = col;
1554         /*
1555          * Valid bit set - map the collection.
1556          * Valid bit cleared - unmap the collection.
1557          */
1558         desc.cmd_desc_mapc.valid = valid;
1559
1560         its_cmd_send(dev, &desc);
1561 }
1562
1563 static void
1564 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
1565 {
1566         struct gicv3_its_softc *sc;
1567         struct its_cmd_desc desc;
1568         struct its_col *col;
1569         u_int col_id;
1570
1571         sc = device_get_softc(dev);
1572
1573         col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
1574         col = sc->sc_its_cols[col_id];
1575
1576         desc.cmd_type = ITS_CMD_MAPTI;
1577         desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
1578         desc.cmd_desc_mapvi.col = col;
1579         /* The EventID sent to the device */
1580         desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base;
1581         /* The physical interrupt presented to softeware */
1582         desc.cmd_desc_mapvi.pid = girq->gi_irq + sc->sc_irq_base;
1583
1584         its_cmd_send(dev, &desc);
1585 }
1586
1587 static void
1588 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
1589 {
1590         struct its_cmd_desc desc;
1591
1592         desc.cmd_type = ITS_CMD_MAPD;
1593         desc.cmd_desc_mapd.its_dev = its_dev;
1594         desc.cmd_desc_mapd.valid = valid;
1595
1596         its_cmd_send(dev, &desc);
1597 }
1598
1599 static void
1600 its_cmd_inv(device_t dev, struct its_dev *its_dev,
1601     struct gicv3_its_irqsrc *girq)
1602 {
1603         struct gicv3_its_softc *sc;
1604         struct its_cmd_desc desc;
1605         struct its_col *col;
1606
1607         sc = device_get_softc(dev);
1608         col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
1609
1610         desc.cmd_type = ITS_CMD_INV;
1611         /* The EventID sent to the device */
1612         desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base;
1613         desc.cmd_desc_inv.its_dev = its_dev;
1614         desc.cmd_desc_inv.col = col;
1615
1616         its_cmd_send(dev, &desc);
1617 }
1618
1619 static void
1620 its_cmd_invall(device_t dev, struct its_col *col)
1621 {
1622         struct its_cmd_desc desc;
1623
1624         desc.cmd_type = ITS_CMD_INVALL;
1625         desc.cmd_desc_invall.col = col;
1626
1627         its_cmd_send(dev, &desc);
1628 }
1629
1630 #ifdef FDT
1631 static device_probe_t gicv3_its_fdt_probe;
1632 static device_attach_t gicv3_its_fdt_attach;
1633
1634 static device_method_t gicv3_its_fdt_methods[] = {
1635         /* Device interface */
1636         DEVMETHOD(device_probe,         gicv3_its_fdt_probe),
1637         DEVMETHOD(device_attach,        gicv3_its_fdt_attach),
1638
1639         /* End */
1640         DEVMETHOD_END
1641 };
1642
1643 #define its_baseclasses its_fdt_baseclasses
1644 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
1645     sizeof(struct gicv3_its_softc), gicv3_its_driver);
1646 #undef its_baseclasses
1647 static devclass_t gicv3_its_fdt_devclass;
1648
1649 EARLY_DRIVER_MODULE(its, gic, gicv3_its_fdt_driver,
1650     gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
1651
1652 static int
1653 gicv3_its_fdt_probe(device_t dev)
1654 {
1655
1656         if (!ofw_bus_status_okay(dev))
1657                 return (ENXIO);
1658
1659         if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
1660                 return (ENXIO);
1661
1662         device_set_desc(dev, "ARM GIC Interrupt Translation Service");
1663         return (BUS_PROBE_DEFAULT);
1664 }
1665
1666 static int
1667 gicv3_its_fdt_attach(device_t dev)
1668 {
1669         struct gicv3_its_softc *sc;
1670         phandle_t xref;
1671         int err;
1672
1673         sc = device_get_softc(dev);
1674
1675         sc->sc_irq_length = gicv3_get_nirqs(dev);
1676         sc->sc_irq_base = GIC_FIRST_LPI;
1677         sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
1678
1679         err = gicv3_its_attach(dev);
1680         if (err != 0)
1681                 return (err);
1682
1683         /* Register this device as a interrupt controller */
1684         xref = OF_xref_from_node(ofw_bus_get_node(dev));
1685         sc->sc_pic = intr_pic_register(dev, xref);
1686         intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
1687             gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
1688
1689         /* Register this device to handle MSI interrupts */
1690         intr_msi_register(dev, xref);
1691
1692         return (0);
1693 }
1694 #endif