sys/sparc64/pci/fire.c

   1 /*-
   2  * Copyright (c) 1999, 2000 Matthew R. Green
   3  * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>
   4  * Copyright (c) 2009 by Marius Strobl <marius@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The name of the author may not be used to endorse or promote products
  16  *    derived from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  *      from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp
  31  *      from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 /*
  38  * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express
  39  * bridges
  40  */
  41
  42 #include "opt_fire.h"
  43 #include "opt_ofw_pci.h"
  44
  45 #include <sys/param.h>
  46 #include <sys/systm.h>
  47 #include <sys/bus.h>
  48 #include <sys/interrupt.h>
  49 #include <sys/kernel.h>
  50 #include <sys/lock.h>
  51 #include <sys/malloc.h>
  52 #include <sys/module.h>
  53 #include <sys/mutex.h>
  54 #include <sys/pciio.h>
  55 #include <sys/pcpu.h>
  56 #include <sys/rman.h>
  57 #include <sys/smp.h>
  58 #include <sys/sysctl.h>
  59 #include <sys/timetc.h>
  60
  61 #include <dev/ofw/ofw_bus.h>
  62 #include <dev/ofw/openfirm.h>
  63
  64 #include <vm/vm.h>
  65 #include <vm/pmap.h>
  66
  67 #include <machine/bus.h>
  68 #include <machine/bus_common.h>
  69 #include <machine/bus_private.h>
  70 #include <machine/iommureg.h>
  71 #include <machine/iommuvar.h>
  72 #include <machine/pmap.h>
  73 #include <machine/resource.h>
  74
  75 #include <dev/pci/pcireg.h>
  76 #include <dev/pci/pcivar.h>
  77
  78 #include <sparc64/pci/ofw_pci.h>
  79 #include <sparc64/pci/firereg.h>
  80 #include <sparc64/pci/firevar.h>
  81
  82 #include "pcib_if.h"
  83
  84 struct fire_msiqarg;
  85
  86 static const struct fire_desc *fire_get_desc(device_t dev);
  87 static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
  88     bus_dmasync_op_t op);
  89 static int fire_get_intrmap(struct fire_softc *sc, u_int ino,
  90     bus_addr_t *intrmapptr, bus_addr_t *intrclrptr);
  91 static void fire_intr_assign(void *arg);
  92 static void fire_intr_clear(void *arg);
  93 static void fire_intr_disable(void *arg);
  94 static void fire_intr_enable(void *arg);
  95 static int fire_intr_register(struct fire_softc *sc, u_int ino);
  96 static inline void fire_msiq_common(struct intr_vector *iv,
  97     struct fire_msiqarg *fmqa);
  98 static void fire_msiq_filter(void *cookie);
  99 static void fire_msiq_handler(void *cookie);
 100 static void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
 101     driver_filter_t handler, void *arg);
 102 static timecounter_get_t fire_get_timecount;
 103
 104 /* Interrupt handlers */
 105 static driver_filter_t fire_dmc_pec;
 106 static driver_filter_t fire_pcie;
 107 static driver_filter_t fire_xcb;
 108
 109 /*
 110  * Methods
 111  */
 112 static pcib_alloc_msi_t fire_alloc_msi;
 113 static pcib_alloc_msix_t fire_alloc_msix;
 114 static bus_alloc_resource_t fire_alloc_resource;
 115 static device_attach_t fire_attach;
 116 static pcib_map_msi_t fire_map_msi;
 117 static pcib_maxslots_t fire_maxslots;
 118 static device_probe_t fire_probe;
 119 static pcib_read_config_t fire_read_config;
 120 static pcib_release_msi_t fire_release_msi;
 121 static pcib_release_msix_t fire_release_msix;
 122 static pcib_route_interrupt_t fire_route_interrupt;
 123 static bus_setup_intr_t fire_setup_intr;
 124 static bus_teardown_intr_t fire_teardown_intr;
 125 static pcib_write_config_t fire_write_config;
 126
 127 static device_method_t fire_methods[] = {
 128         /* Device interface */
 129         DEVMETHOD(device_probe,         fire_probe),
 130         DEVMETHOD(device_attach,        fire_attach),
 131         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 132         DEVMETHOD(device_suspend,       bus_generic_suspend),
 133         DEVMETHOD(device_resume,        bus_generic_resume),
 134
 135         /* Bus interface */
 136         DEVMETHOD(bus_read_ivar,        ofw_pci_read_ivar),
 137         DEVMETHOD(bus_setup_intr,       fire_setup_intr),
 138         DEVMETHOD(bus_teardown_intr,    fire_teardown_intr),
 139         DEVMETHOD(bus_alloc_resource,   fire_alloc_resource),
 140         DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
 141         DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
 142         DEVMETHOD(bus_adjust_resource,  ofw_pci_adjust_resource),
 143         DEVMETHOD(bus_release_resource, bus_generic_release_resource),
 144         DEVMETHOD(bus_get_dma_tag,      ofw_pci_get_dma_tag),
 145
 146         /* pcib interface */
 147         DEVMETHOD(pcib_maxslots,        fire_maxslots),
 148         DEVMETHOD(pcib_read_config,     fire_read_config),
 149         DEVMETHOD(pcib_write_config,    fire_write_config),
 150         DEVMETHOD(pcib_route_interrupt, fire_route_interrupt),
 151         DEVMETHOD(pcib_alloc_msi,       fire_alloc_msi),
 152         DEVMETHOD(pcib_release_msi,     fire_release_msi),
 153         DEVMETHOD(pcib_alloc_msix,      fire_alloc_msix),
 154         DEVMETHOD(pcib_release_msix,    fire_release_msix),
 155         DEVMETHOD(pcib_map_msi,         fire_map_msi),
 156
 157         /* ofw_bus interface */
 158         DEVMETHOD(ofw_bus_get_node,     ofw_pci_get_node),
 159
 160         DEVMETHOD_END
 161 };
 162
 163 static devclass_t fire_devclass;
 164
 165 DEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc));
 166 EARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0,
 167     BUS_PASS_BUS);
 168 MODULE_DEPEND(fire, nexus, 1, 1, 1);
 169
 170 static const struct intr_controller fire_ic = {
 171         fire_intr_enable,
 172         fire_intr_disable,
 173         fire_intr_assign,
 174         fire_intr_clear
 175 };
 176
 177 struct fire_icarg {
 178         struct fire_softc       *fica_sc;
 179         bus_addr_t              fica_map;
 180         bus_addr_t              fica_clr;
 181 };
 182
 183 static const struct intr_controller fire_msiqc_filter = {
 184         fire_intr_enable,
 185         fire_intr_disable,
 186         fire_intr_assign,
 187         NULL
 188 };
 189
 190 struct fire_msiqarg {
 191         struct fire_icarg       fmqa_fica;
 192         struct mtx              fmqa_mtx;
 193         struct fo_msiq_record   *fmqa_base;
 194         uint64_t                fmqa_head;
 195         uint64_t                fmqa_tail;
 196         uint32_t                fmqa_msiq;
 197         uint32_t                fmqa_msi;
 198 };
 199
 200 #define FIRE_PERF_CNT_QLTY      100
 201
 202 #define FIRE_SPC_BARRIER(spc, sc, offs, len, flags)                     \
 203         bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags))
 204 #define FIRE_SPC_READ_8(spc, sc, offs)                                  \
 205         bus_read_8((sc)->sc_mem_res[(spc)], (offs))
 206 #define FIRE_SPC_WRITE_8(spc, sc, offs, v)                              \
 207         bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v))
 208
 209 #ifndef FIRE_DEBUG
 210 #define FIRE_SPC_SET(spc, sc, offs, reg, v)                             \
 211         FIRE_SPC_WRITE_8((spc), (sc), (offs), (v))
 212 #else
 213 #define FIRE_SPC_SET(spc, sc, offs, reg, v) do {                        \
 214         device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n",    \
 215             (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)),   \
 216             (unsigned long long)(v));                                   \
 217         FIRE_SPC_WRITE_8((spc), (sc), (offs), (v));                     \
 218         } while (0)
 219 #endif
 220
 221 #define FIRE_PCI_BARRIER(sc, offs, len, flags)                          \
 222         FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags)
 223 #define FIRE_PCI_READ_8(sc, offs)                                       \
 224         FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs))
 225 #define FIRE_PCI_WRITE_8(sc, offs, v)                                   \
 226         FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v))
 227 #define FIRE_CTRL_BARRIER(sc, offs, len, flags)                         \
 228         FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags)
 229 #define FIRE_CTRL_READ_8(sc, offs)                                      \
 230         FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs))
 231 #define FIRE_CTRL_WRITE_8(sc, offs, v)                                  \
 232         FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v))
 233
 234 #define FIRE_PCI_SET(sc, offs, v)                                       \
 235         FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v))
 236 #define FIRE_CTRL_SET(sc, offs, v)                                      \
 237         FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v))
 238
 239 struct fire_desc {
 240         const char      *fd_string;
 241         int             fd_mode;
 242         const char      *fd_name;
 243 };
 244
 245 static const struct fire_desc fire_compats[] = {
 246         { "pciex108e,80f0",     FIRE_MODE_FIRE,         "Fire" },
 247 #if 0
 248         { "pciex108e,80f8",     FIRE_MODE_OBERON,       "Oberon" },
 249 #endif
 250         { NULL,                 0,                      NULL }
 251 };
 252
 253 static const struct fire_desc *
 254 fire_get_desc(device_t dev)
 255 {
 256         const struct fire_desc *desc;
 257         const char *compat;
 258
 259         compat = ofw_bus_get_compat(dev);
 260         if (compat == NULL)
 261                 return (NULL);
 262         for (desc = fire_compats; desc->fd_string != NULL; desc++)
 263                 if (strcmp(desc->fd_string, compat) == 0)
 264                         return (desc);
 265         return (NULL);
 266 }
 267
 268 static int
 269 fire_probe(device_t dev)
 270 {
 271         const char *dtype;
 272
 273         dtype = ofw_bus_get_type(dev);
 274         if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 &&
 275             fire_get_desc(dev) != NULL) {
 276                 device_set_desc(dev, "Sun Host-PCIe bridge");
 277                 return (BUS_PROBE_GENERIC);
 278         }
 279         return (ENXIO);
 280 }
 281
 282 static int
 283 fire_attach(device_t dev)
 284 {
 285         struct fire_softc *sc;
 286         const struct fire_desc *desc;
 287         struct ofw_pci_msi_ranges msi_ranges;
 288         struct ofw_pci_msi_addr_ranges msi_addr_ranges;
 289         struct ofw_pci_msi_eq_to_devino msi_eq_to_devino;
 290         struct fire_msiqarg *fmqa;
 291         struct timecounter *tc;
 292         bus_dma_tag_t dmat;
 293         uint64_t ino_bitmap, val;
 294         phandle_t node;
 295         uint32_t prop, prop_array[2];
 296         int i, j, mode;
 297         u_int lw;
 298         uint16_t mps;
 299
 300         sc = device_get_softc(dev);
 301         node = ofw_bus_get_node(dev);
 302         desc = fire_get_desc(dev);
 303         mode = desc->fd_mode;
 304
 305         sc->sc_dev = dev;
 306         sc->sc_mode = mode;
 307         sc->sc_flags = 0;
 308
 309         mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF);
 310         mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN);
 311
 312         /*
 313          * Fire and Oberon have two register banks:
 314          * (0) per-PBM PCI Express configuration and status registers
 315          * (1) (shared) Fire/Oberon controller configuration and status
 316          *     registers
 317          */
 318         for (i = 0; i < FIRE_NREG; i++) {
 319                 j = i;
 320                 sc->sc_mem_res[i] = bus_alloc_resource_any(dev,
 321                     SYS_RES_MEMORY, &j, RF_ACTIVE);
 322                 if (sc->sc_mem_res[i] == NULL)
 323                         panic("%s: could not allocate register bank %d",
 324                             __func__, i);
 325         }
 326
 327         if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1)
 328                 panic("%s: could not determine IGN", __func__);
 329         if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1)
 330                 panic("%s: could not determine module-revision", __func__);
 331
 332         device_printf(dev, "%s, module-revision %d, IGN %#x\n",
 333             desc->fd_name, prop, sc->sc_ign);
 334
 335         /*
 336          * Hunt through all the interrupt mapping regs and register
 337          * the interrupt controller for our interrupt vectors.  We do
 338          * this early in order to be able to catch stray interrupts.
 339          */
 340         i = OF_getprop(node, "ino-bitmap", (void *)prop_array,
 341             sizeof(prop_array));
 342         if (i == -1)
 343                 panic("%s: could not get ino-bitmap", __func__);
 344         ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0];
 345         for (i = 0; i <= FO_MAX_INO; i++) {
 346                 if ((ino_bitmap & (1ULL << i)) == 0)
 347                         continue;
 348                 j = fire_intr_register(sc, i);
 349                 if (j != 0)
 350                         device_printf(dev, "could not register interrupt "
 351                             "controller for INO %d (%d)\n", i, j);
 352         }
 353
 354         /* JBC/UBC module initialization */
 355         FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL);
 356         FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
 357         /* not enabled by OpenSolaris */
 358         FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL);
 359         if (sc->sc_mode == FIRE_MODE_FIRE) {
 360                 FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL,
 361                     FIRE_JBUS_PAR_CTRL_P_EN);
 362                 FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN,
 363                     ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) &
 364                     FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) |
 365                     FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT |
 366                     FIRE_JBC_FATAL_RST_EN_CPE_P_INT |
 367                     FIRE_JBC_FATAL_RST_EN_APE_P_INT |
 368                     FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT |
 369                     FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT |
 370                     FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT |
 371                     FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT);
 372                 FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL);
 373         }
 374
 375         /* TLU initialization */
 376         FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR,
 377             FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
 378         /* not enabled by OpenSolaris */
 379         FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN,
 380             FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK);
 381         FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR,
 382             FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
 383         /* not enabled by OpenSolaris */
 384         FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN,
 385             FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK);
 386         FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR,
 387             FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
 388         /* not enabled by OpenSolaris */
 389         FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN,
 390             FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK);
 391         val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) |
 392             ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) &
 393             FO_PCI_TLU_CTRL_L0S_TIM_MASK) |
 394             ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) &
 395             FO_PCI_TLU_CTRL_CFG_MASK);
 396         if (sc->sc_mode == FIRE_MODE_OBERON)
 397                 val &= ~FO_PCI_TLU_CTRL_NWPR_EN;
 398         val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET;
 399         FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val);
 400         FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0);
 401         FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK);
 402
 403         /* DLU/LPU initialization */
 404         if (sc->sc_mode == FIRE_MODE_OBERON)
 405                 FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0);
 406         else
 407                 FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0);
 408         FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG,
 409             FO_PCI_LPU_LNK_LYR_CFG_VC0_EN);
 410         FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL,
 411             FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN |
 412             FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN);
 413         if (sc->sc_mode == FIRE_MODE_OBERON)
 414                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
 415                     (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT <<
 416                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
 417                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
 418         else {
 419                 switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) &
 420                     FO_PCI_TLU_LNK_STAT_WDTH_MASK) >>
 421                     FO_PCI_TLU_LNK_STAT_WDTH_SHFT) {
 422                 case 1:
 423                         lw = 0;
 424                         break;
 425                 case 4:
 426                         lw = 1;
 427                         break;
 428                 case 8:
 429                         lw = 2;
 430                         break;
 431                 case 16:
 432                         lw = 3;
 433                         break;
 434                 default:
 435                         lw = 0;
 436                 }
 437                 mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) &
 438                     FO_PCI_TLU_CTRL_CFG_MPS_MASK) >>
 439                     FO_PCI_TLU_CTRL_CFG_MPS_SHFT;
 440                 i = sizeof(fire_freq_nak_tmr_thrs) /
 441                     sizeof(*fire_freq_nak_tmr_thrs);
 442                 if (mps >= i)
 443                         mps = i - 1;
 444                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS,
 445                     (fire_freq_nak_tmr_thrs[mps][lw] <<
 446                     FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) &
 447                     FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK);
 448                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS,
 449                     (fire_rply_tmr_thrs[mps][lw] <<
 450                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) &
 451                     FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK);
 452                 FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR,
 453                     ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT <<
 454                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) &
 455                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) |
 456                     ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT <<
 457                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) &
 458                     FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK));
 459                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2,
 460                     (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT <<
 461                     FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) &
 462                     FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK);
 463                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3,
 464                     (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT <<
 465                     FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) &
 466                     FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK);
 467                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4,
 468                     ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT <<
 469                     FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) &
 470                     FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) |
 471                     ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT <<
 472                     FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) &
 473                     FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK));
 474                 FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0);
 475         }
 476
 477         /* ILU initialization */
 478         FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL);
 479         /* not enabled by OpenSolaris */
 480         FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL);
 481
 482         /* IMU initialization */
 483         FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL);
 484         FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN,
 485             FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) &
 486             ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S |
 487             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S |
 488             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S |
 489             FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
 490             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
 491             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P));
 492
 493         /* MMU initialization */
 494         FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR,
 495             FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
 496         /* not enabled by OpenSolaris */
 497         FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN,
 498             FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK);
 499
 500         /* DMC initialization */
 501         FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL);
 502         FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0);
 503         FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0);
 504
 505         /* PEC initialization */
 506         FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL);
 507
 508         /* Establish handlers for interesting interrupts. */
 509         if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0)
 510                 fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc);
 511         if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0)
 512                 fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc);
 513
 514         /* MSI/MSI-X support */
 515         if (OF_getprop(node, "#msi", &sc->sc_msi_count,
 516             sizeof(sc->sc_msi_count)) == -1)
 517                 panic("%s: could not determine MSI count", __func__);
 518         if (OF_getprop(node, "msi-ranges", &msi_ranges,
 519             sizeof(msi_ranges)) == -1)
 520                 sc->sc_msi_first = 0;
 521         else
 522                 sc->sc_msi_first = msi_ranges.first;
 523         if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask,
 524             sizeof(sc->sc_msi_data_mask)) == -1)
 525                 panic("%s: could not determine MSI data mask", __func__);
 526         if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width,
 527             sizeof(sc->sc_msix_data_width)) > 0)
 528                 sc->sc_flags |= FIRE_MSIX;
 529         if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges,
 530             sizeof(msi_addr_ranges)) == -1)
 531                 panic("%s: could not determine MSI address ranges", __func__);
 532         sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges);
 533         sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges);
 534         if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count,
 535             sizeof(sc->sc_msiq_count)) == -1)
 536                 panic("%s: could not determine MSI event queue count",
 537                     __func__);
 538         if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size,
 539             sizeof(sc->sc_msiq_size)) == -1)
 540                 panic("%s: could not determine MSI event queue size",
 541                     __func__);
 542         if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino,
 543             sizeof(msi_eq_to_devino)) == -1 &&
 544             OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino,
 545             sizeof(msi_eq_to_devino)) == -1) {
 546                 sc->sc_msiq_first = 0;
 547                 sc->sc_msiq_ino_first = FO_EQ_FIRST_INO;
 548         } else {
 549                 sc->sc_msiq_first = msi_eq_to_devino.eq_first;
 550                 sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first;
 551         }
 552         if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO ||
 553             sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO)
 554                 panic("%s: event queues exceed INO range", __func__);
 555         sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY,
 556             M_DEVBUF, M_NOWAIT | M_ZERO);
 557         if (sc->sc_msi_bitmap == NULL)
 558                 panic("%s: could not malloc MSI bitmap", __func__);
 559         sc->sc_msi_msiq_table = malloc(sc->sc_msi_count *
 560             sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO);
 561         if (sc->sc_msi_msiq_table == NULL)
 562                 panic("%s: could not malloc MSI-MSI event queue table",
 563                     __func__);
 564         sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY,
 565             M_DEVBUF, M_NOWAIT | M_ZERO);
 566         if (sc->sc_msiq_bitmap == NULL)
 567                 panic("%s: could not malloc MSI event queue bitmap", __func__);
 568         j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count;
 569         sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL,
 570             FO_EQ_ALIGNMENT, 0);
 571         if (sc->sc_msiq == NULL)
 572                 panic("%s: could not contigmalloc MSI event queue", __func__);
 573         memset(sc->sc_msiq, 0, j);
 574         FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS |
 575             (pmap_kextract((vm_offset_t)sc->sc_msiq) &
 576             FO_PCI_EQ_BASE_ADDR_MASK));
 577         for (i = 0; i < sc->sc_msi_count; i++) {
 578                 j = (i + sc->sc_msi_first) << 3;
 579                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j,
 580                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) &
 581                     ~FO_PCI_MSI_MAP_V);
 582         }
 583         for (i = 0; i < sc->sc_msiq_count; i++) {
 584                 j = i + sc->sc_msiq_ino_first;
 585                 if ((ino_bitmap & (1ULL << j)) == 0) {
 586                         mtx_lock(&sc->sc_msi_mtx);
 587                         setbit(sc->sc_msiq_bitmap, i);
 588                         mtx_unlock(&sc->sc_msi_mtx);
 589                 }
 590                 fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg;
 591                 mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN);
 592                 fmqa->fmqa_base =
 593                     (struct fo_msiq_record *)((caddr_t)sc->sc_msiq +
 594                     (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i));
 595                 j = i + sc->sc_msiq_first;
 596                 fmqa->fmqa_msiq = j;
 597                 j <<= 3;
 598                 fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j;
 599                 fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j;
 600                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j,
 601                     FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
 602                     FO_PCI_EQ_CTRL_CLR_DIS);
 603                 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail,
 604                     (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
 605                 FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head,
 606                     (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
 607         }
 608         FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 &
 609             FO_PCI_MSI_32_BIT_ADDR_MASK);
 610         FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 &
 611             FO_PCI_MSI_64_BIT_ADDR_MASK);
 612
 613         /*
 614          * Establish a handler for interesting PCIe messages and disable
 615          * unintersting ones.
 616          */
 617         mtx_lock(&sc->sc_msi_mtx);
 618         for (i = 0; i < sc->sc_msiq_count; i++) {
 619                 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
 620                         j = i;
 621                         break;
 622                 }
 623         }
 624         if (i == sc->sc_msiq_count) {
 625                 mtx_unlock(&sc->sc_msi_mtx);
 626                 panic("%s: no spare event queue for PCIe messages", __func__);
 627         }
 628         setbit(sc->sc_msiq_bitmap, j);
 629         mtx_unlock(&sc->sc_msi_mtx);
 630         i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first);
 631         if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0)
 632                 panic("%s: failed to add interrupt for PCIe messages",
 633                     __func__);
 634         fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg);
 635         j += sc->sc_msiq_first;
 636         /*
 637          * "Please note that setting the EQNUM field to a value larger than
 638          * 35 will yield unpredictable results."
 639          */
 640         if (j > 35)
 641                 panic("%s: invalid queue for PCIe messages (%d)",
 642                     __func__, j);
 643         FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V |
 644             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
 645         FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V |
 646             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
 647         FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V |
 648             ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK));
 649         FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0);
 650         FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0);
 651         FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3),
 652             FO_PCI_EQ_CTRL_SET_EN);
 653
 654 #define TC_COUNTER_MAX_MASK     0xffffffff
 655
 656         /*
 657          * Setup JBC/UBC performance counter 0 in bus cycle counting
 658          * mode as timecounter.
 659          */
 660         if (device_get_unit(dev) == 0) {
 661                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0);
 662                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0);
 663                 FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL,
 664                     (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) |
 665                     (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT));
 666                 tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO);
 667                 if (tc == NULL)
 668                         panic("%s: could not malloc timecounter", __func__);
 669                 tc->tc_get_timecount = fire_get_timecount;
 670                 tc->tc_counter_mask = TC_COUNTER_MAX_MASK;
 671                 if (OF_getprop(OF_peer(0), "clock-frequency", &prop,
 672                     sizeof(prop)) == -1)
 673                         panic("%s: could not determine clock frequency",
 674                             __func__);
 675                 tc->tc_frequency = prop;
 676                 tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF);
 677                 tc->tc_priv = sc;
 678                 /*
 679                  * Due to initial problems with the JBus-driven performance
 680                  * counters not advancing which might be firmware dependent
 681                  * ensure that it actually works.
 682                  */
 683                 if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0)
 684                         tc->tc_quality = FIRE_PERF_CNT_QLTY;
 685                 else
 686                         tc->tc_quality = -FIRE_PERF_CNT_QLTY;
 687                 tc_init(tc);
 688         }
 689
 690         /*
 691          * Set up the IOMMU.  Both Fire and Oberon have one per PBM, but
 692          * neither has a streaming buffer.
 693          */
 694         memcpy(&sc->sc_dma_methods, &iommu_dma_methods,
 695             sizeof(sc->sc_dma_methods));
 696         sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM;
 697         if (sc->sc_mode == FIRE_MODE_OBERON) {
 698                 sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE;
 699                 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS);
 700         } else {
 701                 sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync;
 702                 sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS);
 703         }
 704         sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0;
 705         /* Punch in our copies. */
 706         sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]);
 707         sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]);
 708         sc->sc_is.is_iommu = FO_PCI_MMU;
 709         val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL);
 710         iommu_init(device_get_nameunit(dev), &sc->sc_is, 7, -1, 0);
 711 #ifdef FIRE_DEBUG
 712         device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n",
 713             (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr);
 714 #endif
 715         /* Create our DMA tag. */
 716         if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000,
 717             sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr,
 718             0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0)
 719                 panic("%s: could not create PCI DMA tag", __func__);
 720         dmat->dt_cookie = &sc->sc_is;
 721         dmat->dt_mt = &sc->sc_dma_methods;
 722
 723         if (ofw_pci_attach_common(dev, dmat, FO_IO_SIZE, FO_MEM_SIZE) != 0)
 724                 panic("%s: ofw_pci_attach_common() failed", __func__);
 725
 726 #define FIRE_SYSCTL_ADD_UINT(name, arg, desc)                           \
 727         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),                     \
 728             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO,     \
 729             (name), CTLFLAG_RD, (arg), 0, (desc))
 730
 731         FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err,
 732             "ILU unknown errors");
 733         FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async,
 734             "JBC correctable errors");
 735         FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int,
 736             "JBC unsolicited interrupt ACK/NACK errors");
 737         FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd,
 738             "JBC unsolicited read response errors");
 739         FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors");
 740         FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce,
 741             "DLU/TLU correctable errors");
 742         FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal",
 743             &sc->sc_stats_tlu_oe_non_fatal,
 744             "DLU/TLU other event non-fatal errors summary"),
 745         FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err,
 746             "DLU/TLU receive other event errors"),
 747         FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err,
 748             "DLU/TLU transmit other event errors"),
 749         FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue,
 750             "UBC DMARDUE erros");
 751
 752 #undef FIRE_SYSCTL_ADD_UINT
 753
 754         device_add_child(dev, "pci", -1);
 755         return (bus_generic_attach(dev));
 756 }
 757
 758 static void
 759 fire_set_intr(struct fire_softc *sc, u_int index, u_int ino,
 760     driver_filter_t handler, void *arg)
 761 {
 762         u_long vec;
 763         int rid;
 764
 765         rid = index;
 766         sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev,
 767             SYS_RES_IRQ, &rid, RF_ACTIVE);
 768         if (sc->sc_irq_res[index] == NULL ||
 769             INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino ||
 770             INTIGN(vec) != sc->sc_ign ||
 771             intr_vectors[vec].iv_ic != &fire_ic ||
 772             bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index],
 773             INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg,
 774             &sc->sc_ihand[index]) != 0)
 775                 panic("%s: failed to set up interrupt %d", __func__, index);
 776 }
 777
 778 static int
 779 fire_intr_register(struct fire_softc *sc, u_int ino)
 780 {
 781         struct fire_icarg *fica;
 782         bus_addr_t intrclr, intrmap;
 783         int error;
 784
 785         if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0)
 786                 return (ENXIO);
 787         fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ?
 788             sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF,
 789             M_NOWAIT | M_ZERO);
 790         if (fica == NULL)
 791                 return (ENOMEM);
 792         fica->fica_sc = sc;
 793         fica->fica_map = intrmap;
 794         fica->fica_clr = intrclr;
 795         error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino),
 796             &fire_ic, fica));
 797         if (error != 0)
 798                 free(fica, M_DEVBUF);
 799         return (error);
 800 }
 801
 802 static int
 803 fire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr,
 804     bus_addr_t *intrclrptr)
 805 {
 806
 807         if (ino > FO_MAX_INO) {
 808                 device_printf(sc->sc_dev, "out of range INO %d requested\n",
 809                     ino);
 810                 return (0);
 811         }
 812
 813         ino <<= 3;
 814         if (intrmapptr != NULL)
 815                 *intrmapptr = FO_PCI_INT_MAP_BASE + ino;
 816         if (intrclrptr != NULL)
 817                 *intrclrptr = FO_PCI_INT_CLR_BASE + ino;
 818         return (1);
 819 }
 820
 821 /*
 822  * Interrupt handlers
 823  */
 824 static int
 825 fire_dmc_pec(void *arg)
 826 {
 827         struct fire_softc *sc;
 828         device_t dev;
 829         uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar;
 830         uint64_t mmutfsr, oestat, pecstat, uestat, val;
 831         u_int fatal, oenfatal;
 832
 833         fatal = 0;
 834         sc = arg;
 835         dev = sc->sc_dev;
 836         mtx_lock_spin(&sc->sc_pcib_mtx);
 837         mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT);
 838         if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) {
 839                 dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT);
 840                 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) {
 841                         imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT);
 842                         device_printf(dev, "IMU error %#llx\n",
 843                             (unsigned long long)imustat);
 844                         if ((imustat &
 845                             FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) {
 846                                 fatal = 1;
 847                                 val = FIRE_PCI_READ_8(sc,
 848                                     FO_PCI_IMU_SCS_ERR_LOG);
 849                                 device_printf(dev, "SCS error log %#llx\n",
 850                                     (unsigned long long)val);
 851                         }
 852                         if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) {
 853                                 fatal = 1;
 854                                 val = FIRE_PCI_READ_8(sc,
 855                                     FO_PCI_IMU_EQS_ERR_LOG);
 856                                 device_printf(dev, "EQS error log %#llx\n",
 857                                     (unsigned long long)val);
 858                         }
 859                         if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P |
 860                             FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P |
 861                             FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P |
 862                             FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P |
 863                             FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P |
 864                             FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P |
 865                             FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P |
 866                             FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) {
 867                                 fatal = 1;
 868                                 val = FIRE_PCI_READ_8(sc,
 869                                     FO_PCI_IMU_RDS_ERR_LOG);
 870                                 device_printf(dev, "RDS error log %#llx\n",
 871                                     (unsigned long long)val);
 872                         }
 873                 }
 874                 if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) {
 875                         fatal = 1;
 876                         mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT);
 877                         mmutfar = FIRE_PCI_READ_8(sc,
 878                             FO_PCI_MMU_TRANS_FAULT_ADDR);
 879                         mmutfsr = FIRE_PCI_READ_8(sc,
 880                             FO_PCI_MMU_TRANS_FAULT_STAT);
 881                         if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P |
 882                             FO_PCI_MMU_ERR_INT_TBW_ERR_P |
 883                             FO_PCI_MMU_ERR_INT_TBW_UDE_P |
 884                             FO_PCI_MMU_ERR_INT_TBW_DME_P |
 885                             FO_PCI_MMU_ERR_INT_TTC_CAE_P |
 886                             FIRE_PCI_MMU_ERR_INT_TTC_DPE_P |
 887                             OBERON_PCI_MMU_ERR_INT_TTC_DUE_P |
 888                             FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0)
 889                                 fatal = 1;
 890                         else {
 891                                 sc->sc_stats_mmu_err++;
 892                                 FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR,
 893                                     mmustat);
 894                         }
 895                         device_printf(dev,
 896                             "MMU error %#llx: TFAR %#llx TFSR %#llx\n",
 897                             (unsigned long long)mmustat,
 898                             (unsigned long long)mmutfar,
 899                             (unsigned long long)mmutfsr);
 900                 }
 901         }
 902         if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) {
 903                 pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT);
 904                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) {
 905                         fatal = 1;
 906                         uestat = FIRE_PCI_READ_8(sc,
 907                             FO_PCI_TLU_UERR_INT_STAT);
 908                         device_printf(dev,
 909                             "DLU/TLU uncorrectable error %#llx\n",
 910                             (unsigned long long)uestat);
 911                         if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P |
 912                             OBERON_PCI_TLU_UERR_INT_POIS_P |
 913                             FO_PCI_TLU_UERR_INT_MFP_P |
 914                             FO_PCI_TLU_UERR_INT_ROF_P |
 915                             FO_PCI_TLU_UERR_INT_UC_P |
 916                             FIRE_PCI_TLU_UERR_INT_PP_P |
 917                             OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) {
 918                                 val = FIRE_PCI_READ_8(sc,
 919                                     FO_PCI_TLU_RX_UERR_HDR1_LOG);
 920                                 device_printf(dev,
 921                                     "receive header log %#llx\n",
 922                                     (unsigned long long)val);
 923                                 val = FIRE_PCI_READ_8(sc,
 924                                     FO_PCI_TLU_RX_UERR_HDR2_LOG);
 925                                 device_printf(dev,
 926                                     "receive header log 2 %#llx\n",
 927                                     (unsigned long long)val);
 928                         }
 929                         if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) {
 930                                 val = FIRE_PCI_READ_8(sc,
 931                                     FO_PCI_TLU_TX_UERR_HDR1_LOG);
 932                                 device_printf(dev,
 933                                     "transmit header log %#llx\n",
 934                                     (unsigned long long)val);
 935                                 val = FIRE_PCI_READ_8(sc,
 936                                     FO_PCI_TLU_TX_UERR_HDR2_LOG);
 937                                 device_printf(dev,
 938                                     "transmit header log 2 %#llx\n",
 939                                     (unsigned long long)val);
 940                         }
 941                         if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) {
 942                                 val = FIRE_PCI_READ_8(sc,
 943                                     FO_PCI_LPU_LNK_LYR_INT_STAT);
 944                                 device_printf(dev,
 945                                     "link layer interrupt and status %#llx\n",
 946                                     (unsigned long long)val);
 947                         }
 948                         if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) {
 949                                 val = FIRE_PCI_READ_8(sc,
 950                                     FO_PCI_LPU_PHY_LYR_INT_STAT);
 951                                 device_printf(dev,
 952                                     "phy layer interrupt and status %#llx\n",
 953                                     (unsigned long long)val);
 954                         }
 955                 }
 956                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) {
 957                         sc->sc_stats_tlu_ce++;
 958                         cestat = FIRE_PCI_READ_8(sc,
 959                             FO_PCI_TLU_CERR_INT_STAT);
 960                         device_printf(dev,
 961                             "DLU/TLU correctable error %#llx\n",
 962                             (unsigned long long)cestat);
 963                         val = FIRE_PCI_READ_8(sc,
 964                             FO_PCI_LPU_LNK_LYR_INT_STAT);
 965                         device_printf(dev,
 966                             "link layer interrupt and status %#llx\n",
 967                             (unsigned long long)val);
 968                         if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) {
 969                                 FIRE_PCI_WRITE_8(sc,
 970                                     FO_PCI_LPU_LNK_LYR_INT_STAT, val);
 971                                 val = FIRE_PCI_READ_8(sc,
 972                                     FO_PCI_LPU_PHY_LYR_INT_STAT);
 973                                 device_printf(dev,
 974                                     "phy layer interrupt and status %#llx\n",
 975                                     (unsigned long long)val);
 976                         }
 977                         FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR,
 978                             cestat);
 979                 }
 980                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) {
 981                         oenfatal = 0;
 982                         oestat = FIRE_PCI_READ_8(sc,
 983                             FO_PCI_TLU_OEVENT_INT_STAT);
 984                         device_printf(dev, "DLU/TLU other event %#llx\n",
 985                             (unsigned long long)oestat);
 986                         if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
 987                             FO_PCI_TLU_OEVENT_MRC_P |
 988                             FO_PCI_TLU_OEVENT_WUC_P |
 989                             FO_PCI_TLU_OEVENT_RUC_P |
 990                             FO_PCI_TLU_OEVENT_CRS_P)) != 0) {
 991                                 val = FIRE_PCI_READ_8(sc,
 992                                     FO_PCI_TLU_RX_OEVENT_HDR1_LOG);
 993                                 device_printf(dev,
 994                                     "receive header log %#llx\n",
 995                                     (unsigned long long)val);
 996                                 val = FIRE_PCI_READ_8(sc,
 997                                     FO_PCI_TLU_RX_OEVENT_HDR2_LOG);
 998                                 device_printf(dev,
 999                                     "receive header log 2 %#llx\n",
1000                                     (unsigned long long)val);
1001                                 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1002                                     FO_PCI_TLU_OEVENT_MRC_P |
1003                                     FO_PCI_TLU_OEVENT_WUC_P |
1004                                     FO_PCI_TLU_OEVENT_RUC_P)) != 0)
1005                                         fatal = 1;
1006                                 else {
1007                                         sc->sc_stats_tlu_oe_rx_err++;
1008                                         oenfatal = 1;
1009                                 }
1010                         }
1011                         if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1012                             FO_PCI_TLU_OEVENT_CTO_P |
1013                             FO_PCI_TLU_OEVENT_WUC_P |
1014                             FO_PCI_TLU_OEVENT_RUC_P)) != 0) {
1015                                 val = FIRE_PCI_READ_8(sc,
1016                                     FO_PCI_TLU_TX_OEVENT_HDR1_LOG);
1017                                 device_printf(dev,
1018                                     "transmit header log %#llx\n",
1019                                     (unsigned long long)val);
1020                                 val = FIRE_PCI_READ_8(sc,
1021                                     FO_PCI_TLU_TX_OEVENT_HDR2_LOG);
1022                                 device_printf(dev,
1023                                     "transmit header log 2 %#llx\n",
1024                                     (unsigned long long)val);
1025                                 if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P |
1026                                     FO_PCI_TLU_OEVENT_CTO_P |
1027                                     FO_PCI_TLU_OEVENT_WUC_P |
1028                                     FO_PCI_TLU_OEVENT_RUC_P)) != 0)
1029                                         fatal = 1;
1030                                 else {
1031                                         sc->sc_stats_tlu_oe_tx_err++;
1032                                         oenfatal = 1;
1033                                 }
1034                         }
1035                         if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P |
1036                             FO_PCI_TLU_OEVENT_EMP_P |
1037                             FO_PCI_TLU_OEVENT_EPE_P |
1038                             FIRE_PCI_TLU_OEVENT_ERP_P |
1039                             OBERON_PCI_TLU_OEVENT_ERBU_P |
1040                             FIRE_PCI_TLU_OEVENT_EIP_P |
1041                             OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) {
1042                                 fatal = 1;
1043                                 val = FIRE_PCI_READ_8(sc,
1044                                     FO_PCI_LPU_LNK_LYR_INT_STAT);
1045                                 device_printf(dev,
1046                                     "link layer interrupt and status %#llx\n",
1047                                     (unsigned long long)val);
1048                         }
1049                         if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P |
1050                             FO_PCI_TLU_OEVENT_EDP_P |
1051                             FIRE_PCI_TLU_OEVENT_EHP_P |
1052                             OBERON_PCI_TLU_OEVENT_TLUEITMO_S |
1053                             FO_PCI_TLU_OEVENT_ERU_P)) != 0)
1054                                 fatal = 1;
1055                         if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P |
1056                             FO_PCI_TLU_OEVENT_LWC_P |
1057                             FO_PCI_TLU_OEVENT_LIN_P |
1058                             FO_PCI_TLU_OEVENT_LRS_P |
1059                             FO_PCI_TLU_OEVENT_LDN_P |
1060                             FO_PCI_TLU_OEVENT_LUP_P)) != 0)
1061                                 oenfatal = 1;
1062                         if (oenfatal != 0) {
1063                                 sc->sc_stats_tlu_oe_non_fatal++;
1064                                 FIRE_PCI_WRITE_8(sc,
1065                                     FO_PCI_TLU_OEVENT_STAT_CLR, oestat);
1066                                 if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0)
1067                                         FIRE_PCI_WRITE_8(sc,
1068                                             FO_PCI_LPU_LNK_LYR_INT_STAT,
1069                                             FIRE_PCI_READ_8(sc,
1070                                             FO_PCI_LPU_LNK_LYR_INT_STAT));
1071                         }
1072                 }
1073                 if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) {
1074                         ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT);
1075                         device_printf(dev, "ILU error %#llx\n",
1076                             (unsigned long long)ilustat);
1077                         if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P |
1078                             FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0)
1079                             fatal = 1;
1080                         else {
1081                                 sc->sc_stats_ilu_err++;
1082                                 FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT,
1083                                     ilustat);
1084                         }
1085                 }
1086         }
1087         mtx_unlock_spin(&sc->sc_pcib_mtx);
1088         if (fatal != 0)
1089                 panic("%s: fatal DMC/PEC error",
1090                     device_get_nameunit(sc->sc_dev));
1091         return (FILTER_HANDLED);
1092 }
1093
1094 static int
1095 fire_xcb(void *arg)
1096 {
1097         struct fire_softc *sc;
1098         device_t dev;
1099         uint64_t errstat, intstat, val;
1100         u_int fatal;
1101
1102         fatal = 0;
1103         sc = arg;
1104         dev = sc->sc_dev;
1105         mtx_lock_spin(&sc->sc_pcib_mtx);
1106         if (sc->sc_mode == FIRE_MODE_OBERON) {
1107                 intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
1108                 device_printf(dev, "UBC error: interrupt status %#llx\n",
1109                     (unsigned long long)intstat);
1110                 if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P |
1111                     OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0)
1112                         fatal = 1;
1113                 else
1114                         sc->sc_stats_ubc_dmardue++;
1115                 if (fatal != 0) {
1116                         mtx_unlock_spin(&sc->sc_pcib_mtx);
1117                         panic("%s: fatal UBC core block error",
1118                             device_get_nameunit(sc->sc_dev));
1119                 } else {
1120                         FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
1121                         mtx_unlock_spin(&sc->sc_pcib_mtx);
1122                 }
1123         } else {
1124                 errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT);
1125                 if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE |
1126                     FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT |
1127                     FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) {
1128                         intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT);
1129                         device_printf(dev, "JBC interrupt status %#llx\n",
1130                             (unsigned long long)intstat);
1131                         if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) {
1132                                 val = FIRE_CTRL_READ_8(sc,
1133                                     FIRE_JBC_CSR_ERR_LOG);
1134                                 device_printf(dev, "CSR error log %#llx\n",
1135                                     (unsigned long long)val);
1136                         }
1137                         if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P |
1138                             FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) {
1139                                 if ((intstat &
1140                                     FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0)
1141                                         sc->sc_stats_jbc_unsol_rd++;
1142                                 if ((intstat &
1143                                     FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0)
1144                                         sc->sc_stats_jbc_unsol_int++;
1145                                 val = FIRE_CTRL_READ_8(sc,
1146                                     FIRE_DMCINT_IDC_ERR_LOG);
1147                                 device_printf(dev,
1148                                     "DMCINT IDC error log %#llx\n",
1149                                     (unsigned long long)val);
1150                         }
1151                         if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P |
1152                             FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) {
1153                                 fatal = 1;
1154                                 val = FIRE_CTRL_READ_8(sc,
1155                                     FIRE_MERGE_TRANS_ERR_LOG);
1156                                 device_printf(dev,
1157                                     "merge transaction error log %#llx\n",
1158                                     (unsigned long long)val);
1159                         }
1160                         if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) {
1161                                 fatal = 1;
1162                                 val = FIRE_CTRL_READ_8(sc,
1163                                     FIRE_JBCINT_OTRANS_ERR_LOG);
1164                                 device_printf(dev,
1165                                     "JBCINT out transaction error log "
1166                                     "%#llx\n", (unsigned long long)val);
1167                                 val = FIRE_CTRL_READ_8(sc,
1168                                     FIRE_JBCINT_OTRANS_ERR_LOG2);
1169                                 device_printf(dev,
1170                                     "JBCINT out transaction error log 2 "
1171                                     "%#llx\n", (unsigned long long)val);
1172                         }
1173                         if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
1174                             FIRE_JBC_ERR_INT_CE_ASYN_P |
1175                             FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P |
1176                             FIRE_JBC_ERR_INT_JUE_P |
1177                             FIRE_JBC_ERR_INT_ICISE_P |
1178                             FIRE_JBC_ERR_INT_WR_DPE_P |
1179                             FIRE_JBC_ERR_INT_RD_DPE_P |
1180                             FIRE_JBC_ERR_INT_ILL_BMW_P |
1181                             FIRE_JBC_ERR_INT_ILL_BMR_P |
1182                             FIRE_JBC_ERR_INT_BJC_P)) != 0) {
1183                                 if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P |
1184                                     FIRE_JBC_ERR_INT_JTE_P |
1185                                     FIRE_JBC_ERR_INT_JBE_P |
1186                                     FIRE_JBC_ERR_INT_JUE_P |
1187                                     FIRE_JBC_ERR_INT_ICISE_P |
1188                                     FIRE_JBC_ERR_INT_WR_DPE_P |
1189                                     FIRE_JBC_ERR_INT_RD_DPE_P |
1190                                     FIRE_JBC_ERR_INT_ILL_BMW_P |
1191                                     FIRE_JBC_ERR_INT_ILL_BMR_P |
1192                                     FIRE_JBC_ERR_INT_BJC_P)) != 0)
1193                                         fatal = 1;
1194                                 else
1195                                         sc->sc_stats_jbc_ce_async++;
1196                                 val = FIRE_CTRL_READ_8(sc,
1197                                     FIRE_JBCINT_ITRANS_ERR_LOG);
1198                                 device_printf(dev,
1199                                     "JBCINT in transaction error log %#llx\n",
1200                                     (unsigned long long)val);
1201                                 val = FIRE_CTRL_READ_8(sc,
1202                                     FIRE_JBCINT_ITRANS_ERR_LOG2);
1203                                 device_printf(dev,
1204                                     "JBCINT in transaction error log 2 "
1205                                     "%#llx\n", (unsigned long long)val);
1206                         }
1207                         if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P |
1208                             FIRE_JBC_ERR_INT_ILL_ACC_RD_P |
1209                             FIRE_JBC_ERR_INT_PIO_UNMAP_P |
1210                             FIRE_JBC_ERR_INT_PIO_DPE_P |
1211                             FIRE_JBC_ERR_INT_PIO_CPE_P |
1212                             FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) {
1213                                 fatal = 1;
1214                                 val = FIRE_CTRL_READ_8(sc,
1215                                     FIRE_JBC_CSR_ERR_LOG);
1216                                 device_printf(dev,
1217                                     "DMCINT ODCD error log %#llx\n",
1218                                     (unsigned long long)val);
1219                         }
1220                         if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P |
1221                             FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P |
1222                             FIRE_JBC_ERR_INT_PIO_CPE_P |
1223                             FIRE_JBC_ERR_INT_JTCEEW_P |
1224                             FIRE_JBC_ERR_INT_JTCEEI_P |
1225                             FIRE_JBC_ERR_INT_JTCEER_P)) != 0) {
1226                                 fatal = 1;
1227                                 val = FIRE_CTRL_READ_8(sc,
1228                                     FIRE_FATAL_ERR_LOG);
1229                                 device_printf(dev, "fatal error log %#llx\n",
1230                                     (unsigned long long)val);
1231                                 val = FIRE_CTRL_READ_8(sc,
1232                                     FIRE_FATAL_ERR_LOG2);
1233                                 device_printf(dev, "fatal error log 2 "
1234                                     "%#llx\n", (unsigned long long)val);
1235                         }
1236                         if (fatal != 0) {
1237                                 mtx_unlock_spin(&sc->sc_pcib_mtx);
1238                                 panic("%s: fatal JBC core block error",
1239                                     device_get_nameunit(sc->sc_dev));
1240                         } else {
1241                                 FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL);
1242                                 mtx_unlock_spin(&sc->sc_pcib_mtx);
1243                         }
1244                 } else {
1245                         mtx_unlock_spin(&sc->sc_pcib_mtx);
1246                         panic("%s: unknown JCB core block error status %#llx",
1247                             device_get_nameunit(sc->sc_dev),
1248                             (unsigned long long)errstat);
1249                 }
1250         }
1251         return (FILTER_HANDLED);
1252 }
1253
1254 static int
1255 fire_pcie(void *arg)
1256 {
1257         struct fire_msiqarg *fmqa;
1258         struct fire_softc *sc;
1259         struct fo_msiq_record *qrec;
1260         device_t dev;
1261         uint64_t word0;
1262         u_int head, msg, msiq;
1263
1264         fmqa = arg;
1265         sc = fmqa->fmqa_fica.fica_sc;
1266         dev = sc->sc_dev;
1267         msiq = fmqa->fmqa_msiq;
1268         mtx_lock_spin(&fmqa->fmqa_mtx);
1269         head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
1270             FO_PCI_EQ_HD_SHFT;
1271         qrec = &fmqa->fmqa_base[head];
1272         word0 = qrec->fomqr_word0;
1273         for (;;) {
1274                 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0,
1275                     ("%s: received non-PCIe message in event queue %d "
1276                     "(word0 %#llx)", device_get_nameunit(dev), msiq,
1277                     (unsigned long long)word0));
1278                 msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
1279                     FO_MQR_WORD0_DATA0_SHFT;
1280
1281 #define PCIE_MSG_CODE_ERR_COR           0x30
1282 #define PCIE_MSG_CODE_ERR_NONFATAL      0x31
1283 #define PCIE_MSG_CODE_ERR_FATAL         0x33
1284
1285                 if (msg == PCIE_MSG_CODE_ERR_COR)
1286                         device_printf(dev, "correctable PCIe error\n");
1287                 else if (msg == PCIE_MSG_CODE_ERR_NONFATAL ||
1288                     msg == PCIE_MSG_CODE_ERR_FATAL)
1289                         panic("%s: %sfatal PCIe error",
1290                             device_get_nameunit(dev),
1291                             msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : "");
1292                 else
1293                         panic("%s: received unknown PCIe message %#x",
1294                             device_get_nameunit(dev), msg);
1295                 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
1296                 head = (head + 1) % sc->sc_msiq_size;
1297                 qrec = &fmqa->fmqa_base[head];
1298                 word0 = qrec->fomqr_word0;
1299                 if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
1300                         break;
1301         }
1302         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
1303             FO_PCI_EQ_HD_SHFT);
1304         if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
1305             FO_PCI_EQ_TL_OVERR) != 0) {
1306                 device_printf(dev, "event queue %d overflow\n", msiq);
1307                 msiq <<= 3;
1308                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1309                     FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
1310                     FO_PCI_EQ_CTRL_CLR_COVERR);
1311         }
1312         mtx_unlock_spin(&fmqa->fmqa_mtx);
1313         return (FILTER_HANDLED);
1314 }
1315
1316 static int
1317 fire_maxslots(device_t dev)
1318 {
1319
1320         return (1);
1321 }
1322
1323 static uint32_t
1324 fire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
1325     int width)
1326 {
1327
1328         return (ofw_pci_read_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus,
1329             slot, func, reg), bus, slot, func, reg, width));
1330 }
1331
1332 static void
1333 fire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
1334     uint32_t val, int width)
1335 {
1336
1337         ofw_pci_write_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot,
1338             func, reg), bus, slot, func, reg, val, width);
1339 }
1340
1341 static int
1342 fire_route_interrupt(device_t bridge, device_t dev, int pin)
1343 {
1344         ofw_pci_intr_t mintr;
1345
1346         mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
1347         if (!PCI_INTERRUPT_VALID(mintr))
1348                 device_printf(bridge,
1349                     "could not route pin %d for device %d.%d\n",
1350                     pin, pci_get_slot(dev), pci_get_function(dev));
1351         return (mintr);
1352 }
1353
1354 static void
1355 fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
1356     bus_dmasync_op_t op)
1357 {
1358
1359         if ((map->dm_flags & DMF_LOADED) == 0)
1360                 return;
1361
1362         if ((op & BUS_DMASYNC_POSTREAD) != 0)
1363                 ofw_pci_dmamap_sync_stst_order_common();
1364         else if ((op & BUS_DMASYNC_PREWRITE) != 0)
1365                 membar(Sync);
1366 }
1367
1368 static void
1369 fire_intr_enable(void *arg)
1370 {
1371         struct intr_vector *iv;
1372         struct fire_icarg *fica;
1373         struct fire_softc *sc;
1374         struct pcpu *pc;
1375         uint64_t mr;
1376         u_int ctrl, i;
1377
1378         iv = arg;
1379         fica = iv->iv_icarg;
1380         sc = fica->fica_sc;
1381         mr = FO_PCI_IMAP_V;
1382         if (sc->sc_mode == FIRE_MODE_OBERON)
1383                 mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) &
1384                     OBERON_PCI_IMAP_T_DESTID_MASK;
1385         else
1386                 mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) &
1387                     FIRE_PCI_IMAP_T_JPID_MASK;
1388         /*
1389          * Given that all mondos for the same target are required to use the
1390          * same interrupt controller we just use the CPU ID for indexing the
1391          * latter.
1392          */
1393         ctrl = 0;
1394         for (i = 0; i < mp_ncpus; ++i) {
1395                 pc = pcpu_find(i);
1396                 if (pc == NULL || iv->iv_mid != pc->pc_mid)
1397                         continue;
1398                 ctrl = pc->pc_cpuid % 4;
1399                 break;
1400         }
1401         mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT &
1402             FO_PCI_IMAP_INT_CTRL_NUM_MASK;
1403         FIRE_PCI_WRITE_8(sc, fica->fica_map, mr);
1404 }
1405
1406 static void
1407 fire_intr_disable(void *arg)
1408 {
1409         struct intr_vector *iv;
1410         struct fire_icarg *fica;
1411         struct fire_softc *sc;
1412
1413         iv = arg;
1414         fica = iv->iv_icarg;
1415         sc = fica->fica_sc;
1416         FIRE_PCI_WRITE_8(sc, fica->fica_map,
1417             FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V);
1418 }
1419
1420 static void
1421 fire_intr_assign(void *arg)
1422 {
1423         struct intr_vector *iv;
1424         struct fire_icarg *fica;
1425         struct fire_softc *sc;
1426         uint64_t mr;
1427
1428         iv = arg;
1429         fica = iv->iv_icarg;
1430         sc = fica->fica_sc;
1431         mr = FIRE_PCI_READ_8(sc, fica->fica_map);
1432         if ((mr & FO_PCI_IMAP_V) != 0) {
1433                 FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V);
1434                 FIRE_PCI_BARRIER(sc, fica->fica_map, 8,
1435                     BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
1436         }
1437         while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE)
1438                 ;
1439         if ((mr & FO_PCI_IMAP_V) != 0)
1440                 fire_intr_enable(arg);
1441 }
1442
1443 static void
1444 fire_intr_clear(void *arg)
1445 {
1446         struct intr_vector *iv;
1447         struct fire_icarg *fica;
1448
1449         iv = arg;
1450         fica = iv->iv_icarg;
1451         FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE);
1452 }
1453
1454 /*
1455  * Given that the event queue implementation matches our current MD and MI
1456  * interrupt frameworks like square pegs fit into round holes we are generous
1457  * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs
1458  * per Host-PCIe-bridge (we use one event queue for the PCIe error messages).
1459  * This seems tolerable as long as most devices just use one MSI/MSI-X anyway.
1460  * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us
1461  * to decouple the 1:1 mapping at the cost of no longer being able to bind
1462  * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to
1463  * quiesce a device while we move its MSIs/MSI-Xs to another event queue.
1464  */
1465
1466 static int
1467 fire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused,
1468     int *irqs)
1469 {
1470         struct fire_softc *sc;
1471         u_int i, j, msiqrun;
1472
1473         if (powerof2(count) == 0 || count > 32)
1474                 return (EINVAL);
1475
1476         sc = device_get_softc(dev);
1477         mtx_lock(&sc->sc_msi_mtx);
1478         msiqrun = 0;
1479         for (i = 0; i < sc->sc_msiq_count; i++) {
1480                 for (j = i; j < i + count; j++) {
1481                         if (isclr(sc->sc_msiq_bitmap, j) == 0)
1482                                 break;
1483                 }
1484                 if (j == i + count) {
1485                         msiqrun = i;
1486                         break;
1487                 }
1488         }
1489         if (i == sc->sc_msiq_count) {
1490                 mtx_unlock(&sc->sc_msi_mtx);
1491                 return (ENXIO);
1492         }
1493         for (i = 0; i + count < sc->sc_msi_count; i += count) {
1494                 for (j = i; j < i + count; j++)
1495                         if (isclr(sc->sc_msi_bitmap, j) == 0)
1496                                 break;
1497                 if (j == i + count) {
1498                         for (j = 0; j < count; j++) {
1499                                 setbit(sc->sc_msiq_bitmap, msiqrun + j);
1500                                 setbit(sc->sc_msi_bitmap, i + j);
1501                                 sc->sc_msi_msiq_table[i + j] = msiqrun + j;
1502                                 irqs[j] = sc->sc_msi_first + i + j;
1503                         }
1504                         mtx_unlock(&sc->sc_msi_mtx);
1505                         return (0);
1506                 }
1507         }
1508         mtx_unlock(&sc->sc_msi_mtx);
1509         return (ENXIO);
1510 }
1511
1512 static int
1513 fire_release_msi(device_t dev, device_t child, int count, int *irqs)
1514 {
1515         struct fire_softc *sc;
1516         u_int i;
1517
1518         sc = device_get_softc(dev);
1519         mtx_lock(&sc->sc_msi_mtx);
1520         for (i = 0; i < count; i++) {
1521                 clrbit(sc->sc_msiq_bitmap,
1522                     sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]);
1523                 clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first);
1524         }
1525         mtx_unlock(&sc->sc_msi_mtx);
1526         return (0);
1527 }
1528
1529 static int
1530 fire_alloc_msix(device_t dev, device_t child, int *irq)
1531 {
1532         struct fire_softc *sc;
1533         int i, msiq;
1534
1535         sc = device_get_softc(dev);
1536         if ((sc->sc_flags & FIRE_MSIX) == 0)
1537                 return (ENXIO);
1538         mtx_lock(&sc->sc_msi_mtx);
1539         msiq = 0;
1540         for (i = 0; i < sc->sc_msiq_count; i++) {
1541                 if (isclr(sc->sc_msiq_bitmap, i) != 0) {
1542                         msiq = i;
1543                         break;
1544                 }
1545         }
1546         if (i == sc->sc_msiq_count) {
1547                 mtx_unlock(&sc->sc_msi_mtx);
1548                 return (ENXIO);
1549         }
1550         for (i = sc->sc_msi_count - 1; i >= 0; i--) {
1551                 if (isclr(sc->sc_msi_bitmap, i) != 0) {
1552                         setbit(sc->sc_msiq_bitmap, msiq);
1553                         setbit(sc->sc_msi_bitmap, i);
1554                         sc->sc_msi_msiq_table[i] = msiq;
1555                         *irq = sc->sc_msi_first + i;
1556                         mtx_unlock(&sc->sc_msi_mtx);
1557                         return (0);
1558                 }
1559         }
1560         mtx_unlock(&sc->sc_msi_mtx);
1561         return (ENXIO);
1562 }
1563
1564 static int
1565 fire_release_msix(device_t dev, device_t child, int irq)
1566 {
1567         struct fire_softc *sc;
1568
1569         sc = device_get_softc(dev);
1570         if ((sc->sc_flags & FIRE_MSIX) == 0)
1571                 return (ENXIO);
1572         mtx_lock(&sc->sc_msi_mtx);
1573         clrbit(sc->sc_msiq_bitmap,
1574             sc->sc_msi_msiq_table[irq - sc->sc_msi_first]);
1575         clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first);
1576         mtx_unlock(&sc->sc_msi_mtx);
1577         return (0);
1578 }
1579
1580 static int
1581 fire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr,
1582     uint32_t *data)
1583 {
1584         struct fire_softc *sc;
1585         struct pci_devinfo *dinfo;
1586
1587         sc = device_get_softc(dev);
1588         dinfo = device_get_ivars(child);
1589         if (dinfo->cfg.msi.msi_alloc > 0) {
1590                 if ((irq & ~sc->sc_msi_data_mask) != 0) {
1591                         device_printf(dev, "invalid MSI 0x%x\n", irq);
1592                         return (EINVAL);
1593                 }
1594         } else {
1595                 if ((sc->sc_flags & FIRE_MSIX) == 0)
1596                         return (ENXIO);
1597                 if (fls(irq) > sc->sc_msix_data_width) {
1598                         device_printf(dev, "invalid MSI-X 0x%x\n", irq);
1599                         return (EINVAL);
1600                 }
1601         }
1602         if (dinfo->cfg.msi.msi_alloc > 0 &&
1603             (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0)
1604                 *addr = sc->sc_msi_addr32;
1605         else
1606                 *addr = sc->sc_msi_addr64;
1607         *data = irq;
1608         return (0);
1609 }
1610
1611 static void
1612 fire_msiq_handler(void *cookie)
1613 {
1614         struct intr_vector *iv;
1615         struct fire_msiqarg *fmqa;
1616
1617         iv = cookie;
1618         fmqa = iv->iv_icarg;
1619         /*
1620          * Note that since fire_intr_clear() will clear the event queue
1621          * interrupt after the handler associated with the MSI [sic] has
1622          * been executed we have to protect the access to the event queue as
1623          * otherwise nested event queue interrupts cause corruption of the
1624          * event queue on MP machines.  Obviously especially when abandoning
1625          * the 1:1 mapping it would be better to not clear the event queue
1626          * interrupt after each handler invocation but only once when the
1627          * outstanding MSIs have been processed but unfortunately that
1628          * doesn't work well and leads to interrupt storms with controllers/
1629          * drivers which don't mask interrupts while the handler is executed.
1630          * Maybe delaying clearing the MSI until after the handler has been
1631          * executed could be used to work around this but that's not the
1632          * intended usage and might in turn cause lost MSIs.
1633          */
1634         mtx_lock_spin(&fmqa->fmqa_mtx);
1635         fire_msiq_common(iv, fmqa);
1636         mtx_unlock_spin(&fmqa->fmqa_mtx);
1637 }
1638
1639 static void
1640 fire_msiq_filter(void *cookie)
1641 {
1642         struct intr_vector *iv;
1643         struct fire_msiqarg *fmqa;
1644
1645         iv = cookie;
1646         fmqa = iv->iv_icarg;
1647         /*
1648          * For filters we don't use fire_intr_clear() since it would clear
1649          * the event queue interrupt while we're still processing the event
1650          * queue as filters and associated post-filter handler are executed
1651          * directly, which in turn would lead to lost MSIs.  So we clear the
1652          * event queue interrupt only once after processing the event queue.
1653          * Given that this still guarantees the filters to not be executed
1654          * concurrently and no other CPU can clear the event queue interrupt
1655          * while the event queue is still processed, we don't even need to
1656          * interlock the access to the event queue in this case.
1657          */
1658         critical_enter();
1659         fire_msiq_common(iv, fmqa);
1660         FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr,
1661             INTCLR_IDLE);
1662         critical_exit();
1663 }
1664
1665 static inline void
1666 fire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa)
1667 {
1668         struct fire_softc *sc;
1669         struct fo_msiq_record *qrec;
1670         device_t dev;
1671         uint64_t word0;
1672         u_int head, msi, msiq;
1673
1674         sc = fmqa->fmqa_fica.fica_sc;
1675         dev = sc->sc_dev;
1676         msiq = fmqa->fmqa_msiq;
1677         head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >>
1678             FO_PCI_EQ_HD_SHFT;
1679         qrec = &fmqa->fmqa_base[head];
1680         word0 = qrec->fomqr_word0;
1681         for (;;) {
1682                 if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0))
1683                         break;
1684                 KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 ||
1685                     (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0,
1686                     ("%s: received non-MSI/MSI-X message in event queue %d "
1687                     "(word0 %#llx)", device_get_nameunit(dev), msiq,
1688                     (unsigned long long)word0));
1689                 msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >>
1690                     FO_MQR_WORD0_DATA0_SHFT;
1691                 /*
1692                  * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping.
1693                  */
1694                 KASSERT(msi == fmqa->fmqa_msi,
1695                     ("%s: received non-matching MSI/MSI-X in event queue %d "
1696                     "(%d versus %d)", device_get_nameunit(dev), msiq, msi,
1697                     fmqa->fmqa_msi));
1698                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3),
1699                     FO_PCI_MSI_CLR_EQWR_N);
1700                 if (__predict_false(intr_event_handle(iv->iv_event,
1701                     NULL) != 0))
1702                         printf("stray MSI/MSI-X in event queue %d\n", msiq);
1703                 qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK;
1704                 head = (head + 1) % sc->sc_msiq_size;
1705                 qrec = &fmqa->fmqa_base[head];
1706                 word0 = qrec->fomqr_word0;
1707         }
1708         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) <<
1709             FO_PCI_EQ_HD_SHFT);
1710         if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) &
1711             FO_PCI_EQ_TL_OVERR) != 0)) {
1712                 device_printf(dev, "event queue %d overflow\n", msiq);
1713                 msiq <<= 3;
1714                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1715                     FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) |
1716                     FO_PCI_EQ_CTRL_CLR_COVERR);
1717         }
1718 }
1719
1720 static int
1721 fire_setup_intr(device_t dev, device_t child, struct resource *ires,
1722     int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg,
1723     void **cookiep)
1724 {
1725         struct fire_softc *sc;
1726         struct fire_msiqarg *fmqa;
1727         u_long vec;
1728         int error;
1729         u_int msi, msiq;
1730
1731         sc = device_get_softc(dev);
1732         /*
1733          * XXX this assumes that a device only has one INTx, while in fact
1734          * Cassini+ and Saturn can use all four the firmware has assigned
1735          * to them, but so does pci(4).
1736          */
1737         if (rman_get_rid(ires) != 0) {
1738                 msi = rman_get_start(ires);
1739                 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
1740                 vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq);
1741                 msiq += sc->sc_msiq_first;
1742                 if (intr_vectors[vec].iv_ic != &fire_ic) {
1743                         device_printf(dev,
1744                             "invalid interrupt controller for vector 0x%lx\n",
1745                             vec);
1746                         return (EINVAL);
1747                 }
1748                 /*
1749                  * The MD interrupt code needs the vector rather than the MSI.
1750                  */
1751                 rman_set_start(ires, vec);
1752                 rman_set_end(ires, vec);
1753                 error = bus_generic_setup_intr(dev, child, ires, flags, filt,
1754                     intr, arg, cookiep);
1755                 rman_set_start(ires, msi);
1756                 rman_set_end(ires, msi);
1757                 if (error != 0)
1758                         return (error);
1759                 fmqa = intr_vectors[vec].iv_icarg;
1760                 /*
1761                  * XXX inject our event queue handler.
1762                  */
1763                 if (filt != NULL) {
1764                         intr_vectors[vec].iv_func = fire_msiq_filter;
1765                         intr_vectors[vec].iv_ic = &fire_msiqc_filter;
1766                         /*
1767                          * Ensure the event queue interrupt is cleared, it
1768                          * might have triggered before.  Given we supply NULL
1769                          * as ic_clear, inthand_add() won't do this for us.
1770                          */
1771                         FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr,
1772                             INTCLR_IDLE);
1773                 } else
1774                         intr_vectors[vec].iv_func = fire_msiq_handler;
1775                 /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */
1776                 fmqa->fmqa_msi = msi;
1777                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3),
1778                     FO_PCI_EQ_CTRL_SET_EN);
1779                 msi <<= 3;
1780                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1781                     (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
1782                     ~FO_PCI_MSI_MAP_EQNUM_MASK) |
1783                     ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) &
1784                     FO_PCI_MSI_MAP_EQNUM_MASK));
1785                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi,
1786                     FO_PCI_MSI_CLR_EQWR_N);
1787                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1788                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) |
1789                     FO_PCI_MSI_MAP_V);
1790                 return (error);
1791         }
1792
1793         /*
1794          * Make sure the vector is fully specified and we registered
1795          * our interrupt controller for it.
1796          */
1797         vec = rman_get_start(ires);
1798         if (INTIGN(vec) != sc->sc_ign) {
1799                 device_printf(dev, "invalid interrupt vector 0x%lx\n", vec);
1800                 return (EINVAL);
1801         }
1802         if (intr_vectors[vec].iv_ic != &fire_ic) {
1803                 device_printf(dev,
1804                     "invalid interrupt controller for vector 0x%lx\n", vec);
1805                 return (EINVAL);
1806         }
1807         return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr,
1808             arg, cookiep));
1809 }
1810
1811 static int
1812 fire_teardown_intr(device_t dev, device_t child, struct resource *ires,
1813     void *cookie)
1814 {
1815         struct fire_softc *sc;
1816         u_long vec;
1817         int error;
1818         u_int msi, msiq;
1819
1820         sc = device_get_softc(dev);
1821         if (rman_get_rid(ires) != 0) {
1822                 msi = rman_get_start(ires);
1823                 msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first];
1824                 vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first);
1825                 msiq += sc->sc_msiq_first;
1826                 msi <<= 3;
1827                 FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi,
1828                     FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) &
1829                     ~FO_PCI_MSI_MAP_V);
1830                 msiq <<= 3;
1831                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq,
1832                     FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I |
1833                     FO_PCI_EQ_CTRL_CLR_DIS);
1834                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq,
1835                     (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK);
1836                 FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq,
1837                     (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK);
1838                 intr_vectors[vec].iv_ic = &fire_ic;
1839                 /*
1840                  * The MD interrupt code needs the vector rather than the MSI.
1841                  */
1842                 rman_set_start(ires, vec);
1843                 rman_set_end(ires, vec);
1844                 error = bus_generic_teardown_intr(dev, child, ires, cookie);
1845                 msi >>= 3;
1846                 rman_set_start(ires, msi);
1847                 rman_set_end(ires, msi);
1848                 return (error);
1849         }
1850         return (bus_generic_teardown_intr(dev, child, ires, cookie));
1851 }
1852
1853 static struct resource *
1854 fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
1855     u_long start, u_long end, u_long count, u_int flags)
1856 {
1857         struct fire_softc *sc;
1858
1859         if (type == SYS_RES_IRQ && *rid == 0) {
1860                 sc = device_get_softc(bus);
1861                 start = end = INTMAP_VEC(sc->sc_ign, end);
1862         }
1863         return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
1864             count, flags));
1865 }
1866
1867 static u_int
1868 fire_get_timecount(struct timecounter *tc)
1869 {
1870         struct fire_softc *sc;
1871
1872         sc = tc->tc_priv;
1873         return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK);
1874 }