sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58
  59 #include "common/common.h"
  60 #include "common/t4_msg.h"
  61 #include "common/t4_regs.h"
  62 #include "common/t4_regs_values.h"
  63 #include "t4_ioctl.h"
  64 #include "t4_l2t.h"
  65
  66 /* T4 bus driver interface */
  67 static int t4_probe(device_t);
  68 static int t4_attach(device_t);
  69 static int t4_detach(device_t);
  70 static device_method_t t4_methods[] = {
  71         DEVMETHOD(device_probe,         t4_probe),
  72         DEVMETHOD(device_attach,        t4_attach),
  73         DEVMETHOD(device_detach,        t4_detach),
  74
  75         DEVMETHOD_END
  76 };
  77 static driver_t t4_driver = {
  78         "t4nex",
  79         t4_methods,
  80         sizeof(struct adapter)
  81 };
  82
  83
  84 /* T4 port (cxgbe) interface */
  85 static int cxgbe_probe(device_t);
  86 static int cxgbe_attach(device_t);
  87 static int cxgbe_detach(device_t);
  88 static device_method_t cxgbe_methods[] = {
  89         DEVMETHOD(device_probe,         cxgbe_probe),
  90         DEVMETHOD(device_attach,        cxgbe_attach),
  91         DEVMETHOD(device_detach,        cxgbe_detach),
  92         { 0, 0 }
  93 };
  94 static driver_t cxgbe_driver = {
  95         "cxgbe",
  96         cxgbe_methods,
  97         sizeof(struct port_info)
  98 };
  99
 100 static d_ioctl_t t4_ioctl;
 101 static d_open_t t4_open;
 102 static d_close_t t4_close;
 103
 104 static struct cdevsw t4_cdevsw = {
 105        .d_version = D_VERSION,
 106        .d_flags = 0,
 107        .d_open = t4_open,
 108        .d_close = t4_close,
 109        .d_ioctl = t4_ioctl,
 110        .d_name = "t4nex",
 111 };
 112
 113 /* ifnet + media interface */
 114 static void cxgbe_init(void *);
 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 117 static void cxgbe_qflush(struct ifnet *);
 118 static int cxgbe_media_change(struct ifnet *);
 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 120
 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
 122
 123 /*
 124  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 125  * then ADAPTER_LOCK, then t4_uld_list_lock.
 126  */
 127 static struct mtx t4_list_lock;
 128 static SLIST_HEAD(, adapter) t4_list;
 129 #ifdef TCP_OFFLOAD
 130 static struct mtx t4_uld_list_lock;
 131 static SLIST_HEAD(, uld_info) t4_uld_list;
 132 #endif
 133
 134 /*
 135  * Tunables.  See tweak_tunables() too.
 136  */
 137
 138 /*
 139  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 140  */
 141 #define NTXQ_10G 16
 142 static int t4_ntxq10g = -1;
 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 144
 145 #define NRXQ_10G 8
 146 static int t4_nrxq10g = -1;
 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 148
 149 #define NTXQ_1G 4
 150 static int t4_ntxq1g = -1;
 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 152
 153 #define NRXQ_1G 2
 154 static int t4_nrxq1g = -1;
 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 156
 157 #ifdef TCP_OFFLOAD
 158 #define NOFLDTXQ_10G 8
 159 static int t4_nofldtxq10g = -1;
 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 161
 162 #define NOFLDRXQ_10G 2
 163 static int t4_nofldrxq10g = -1;
 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 165
 166 #define NOFLDTXQ_1G 2
 167 static int t4_nofldtxq1g = -1;
 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 169
 170 #define NOFLDRXQ_1G 1
 171 static int t4_nofldrxq1g = -1;
 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 173 #endif
 174
 175 /*
 176  * Holdoff parameters for 10G and 1G ports.
 177  */
 178 #define TMR_IDX_10G 1
 179 static int t4_tmr_idx_10g = TMR_IDX_10G;
 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 181
 182 #define PKTC_IDX_10G (-1)
 183 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 185
 186 #define TMR_IDX_1G 1
 187 static int t4_tmr_idx_1g = TMR_IDX_1G;
 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 189
 190 #define PKTC_IDX_1G (-1)
 191 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 193
 194 /*
 195  * Size (# of entries) of each tx and rx queue.
 196  */
 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 199
 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 202
 203 /*
 204  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 205  */
 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 208
 209 /*
 210  * Configuration file.
 211  */
 212 static char t4_cfg_file[32] = "default";
 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 214
 215 /*
 216  * ASIC features that will be used.  Disable the ones you don't want so that the
 217  * chip resources aren't wasted on features that will not be used.
 218  */
 219 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 220 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 221
 222 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 223 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 224
 225 static int t4_toecaps_allowed = -1;
 226 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 227
 228 static int t4_rdmacaps_allowed = 0;
 229 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 230
 231 static int t4_iscsicaps_allowed = 0;
 232 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 233
 234 static int t4_fcoecaps_allowed = 0;
 235 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 236
 237 struct intrs_and_queues {
 238         int intr_type;          /* INTx, MSI, or MSI-X */
 239         int nirq;               /* Number of vectors */
 240         int intr_flags;
 241         int ntxq10g;            /* # of NIC txq's for each 10G port */
 242         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 243         int ntxq1g;             /* # of NIC txq's for each 1G port */
 244         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 245 #ifdef TCP_OFFLOAD
 246         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 247         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 248         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 249         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 250 #endif
 251 };
 252
 253 struct filter_entry {
 254         uint32_t valid:1;       /* filter allocated and valid */
 255         uint32_t locked:1;      /* filter is administratively locked */
 256         uint32_t pending:1;     /* filter action is pending firmware reply */
 257         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 258         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 259
 260         struct t4_filter_specification fs;
 261 };
 262
 263 enum {
 264         XGMAC_MTU       = (1 << 0),
 265         XGMAC_PROMISC   = (1 << 1),
 266         XGMAC_ALLMULTI  = (1 << 2),
 267         XGMAC_VLANEX    = (1 << 3),
 268         XGMAC_UCADDR    = (1 << 4),
 269         XGMAC_MCADDRS   = (1 << 5),
 270
 271         XGMAC_ALL       = 0xffff
 272 };
 273
 274 static int map_bars(struct adapter *);
 275 static void setup_memwin(struct adapter *);
 276 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 277     struct intrs_and_queues *);
 278 static int prep_firmware(struct adapter *);
 279 static int upload_config_file(struct adapter *, const struct firmware *,
 280     uint32_t *, uint32_t *);
 281 static int partition_resources(struct adapter *, const struct firmware *);
 282 static int get_params__pre_init(struct adapter *);
 283 static int get_params__post_init(struct adapter *);
 284 static void t4_set_desc(struct adapter *);
 285 static void build_medialist(struct port_info *);
 286 static int update_mac_settings(struct port_info *, int);
 287 static int cxgbe_init_locked(struct port_info *);
 288 static int cxgbe_init_synchronized(struct port_info *);
 289 static int cxgbe_uninit_locked(struct port_info *);
 290 static int cxgbe_uninit_synchronized(struct port_info *);
 291 static int adapter_full_init(struct adapter *);
 292 static int adapter_full_uninit(struct adapter *);
 293 static int port_full_init(struct port_info *);
 294 static int port_full_uninit(struct port_info *);
 295 static void quiesce_eq(struct adapter *, struct sge_eq *);
 296 static void quiesce_iq(struct adapter *, struct sge_iq *);
 297 static void quiesce_fl(struct adapter *, struct sge_fl *);
 298 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 299     driver_intr_t *, void *, char *);
 300 static int t4_free_irq(struct adapter *, struct irq *);
 301 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 302     unsigned int);
 303 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 304 static void cxgbe_tick(void *);
 305 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 306 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 307     struct mbuf *);
 308 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 309 static int t4_sysctls(struct adapter *);
 310 static int cxgbe_sysctls(struct port_info *);
 311 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 312 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 313 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 314 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 315 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 316 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 317 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 318 #ifdef SBUF_DRAIN
 319 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 320 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 321 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 322 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 323 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 324 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 325 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 326 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 327 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 328 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 329 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 330 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 331 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 332 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 333 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 334 #endif
 335 static inline void txq_start(struct ifnet *, struct sge_txq *);
 336 static uint32_t fconf_to_mode(uint32_t);
 337 static uint32_t mode_to_fconf(uint32_t);
 338 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 339 static int get_filter_mode(struct adapter *, uint32_t *);
 340 static int set_filter_mode(struct adapter *, uint32_t);
 341 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 342 static int get_filter(struct adapter *, struct t4_filter *);
 343 static int set_filter(struct adapter *, struct t4_filter *);
 344 static int del_filter(struct adapter *, struct t4_filter *);
 345 static void clear_filter(struct filter_entry *);
 346 static int set_filter_wr(struct adapter *, int);
 347 static int del_filter_wr(struct adapter *, int);
 348 static int filter_rpl(struct sge_iq *, const struct rss_header *,
 349     struct mbuf *);
 350 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 351 static int read_card_mem(struct adapter *, struct t4_mem_range *);
 352 #ifdef TCP_OFFLOAD
 353 static int toe_capability(struct port_info *, int);
 354 #endif
 355 static int t4_mod_event(module_t, int, void *);
 356
 357 struct t4_pciids {
 358         uint16_t device;
 359         char *desc;
 360 } t4_pciids[] = {
 361         {0xa000, "Chelsio Terminator 4 FPGA"},
 362         {0x4400, "Chelsio T440-dbg"},
 363         {0x4401, "Chelsio T420-CR"},
 364         {0x4402, "Chelsio T422-CR"},
 365         {0x4403, "Chelsio T440-CR"},
 366         {0x4404, "Chelsio T420-BCH"},
 367         {0x4405, "Chelsio T440-BCH"},
 368         {0x4406, "Chelsio T440-CH"},
 369         {0x4407, "Chelsio T420-SO"},
 370         {0x4408, "Chelsio T420-CX"},
 371         {0x4409, "Chelsio T420-BT"},
 372         {0x440a, "Chelsio T404-BT"},
 373 };
 374
 375 #ifdef TCP_OFFLOAD
 376 /*
 377  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 378  * exactly the same for both rxq and ofld_rxq.
 379  */
 380 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 381 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 382 #endif
 383
 384 static int
 385 t4_probe(device_t dev)
 386 {
 387         int i;
 388         uint16_t v = pci_get_vendor(dev);
 389         uint16_t d = pci_get_device(dev);
 390         uint8_t f = pci_get_function(dev);
 391
 392         if (v != PCI_VENDOR_ID_CHELSIO)
 393                 return (ENXIO);
 394
 395         /* Attach only to PF0 of the FPGA */
 396         if (d == 0xa000 && f != 0)
 397                 return (ENXIO);
 398
 399         for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) {
 400                 if (d == t4_pciids[i].device) {
 401                         device_set_desc(dev, t4_pciids[i].desc);
 402                         return (BUS_PROBE_DEFAULT);
 403                 }
 404         }
 405
 406         return (ENXIO);
 407 }
 408
 409 static int
 410 t4_attach(device_t dev)
 411 {
 412         struct adapter *sc;
 413         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 414         struct intrs_and_queues iaq;
 415         struct sge *s;
 416 #ifdef TCP_OFFLOAD
 417         int ofld_rqidx, ofld_tqidx;
 418 #endif
 419
 420         sc = device_get_softc(dev);
 421         sc->dev = dev;
 422
 423         pci_enable_busmaster(dev);
 424         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 425                 uint32_t v;
 426
 427                 pci_set_max_read_req(dev, 4096);
 428                 v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2);
 429                 v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE;
 430                 pci_write_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, v, 2);
 431         }
 432
 433         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 434             device_get_nameunit(dev));
 435         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 436         mtx_lock(&t4_list_lock);
 437         SLIST_INSERT_HEAD(&t4_list, sc, link);
 438         mtx_unlock(&t4_list_lock);
 439
 440         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 441         TAILQ_INIT(&sc->sfl);
 442         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 443
 444         rc = map_bars(sc);
 445         if (rc != 0)
 446                 goto done; /* error message displayed already */
 447
 448         /*
 449          * This is the real PF# to which we're attaching.  Works from within PCI
 450          * passthrough environments too, where pci_get_function() could return a
 451          * different PF# depending on the passthrough configuration.  We need to
 452          * use the real PF# in all our communication with the firmware.
 453          */
 454         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 455         sc->mbox = sc->pf;
 456
 457         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 458         sc->an_handler = an_not_handled;
 459         for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++)
 460                 sc->cpl_handler[i] = cpl_not_handled;
 461         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl);
 462
 463         /* Prepare the adapter for operation */
 464         rc = -t4_prep_adapter(sc);
 465         if (rc != 0) {
 466                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 467                 goto done;
 468         }
 469
 470         /*
 471          * Do this really early, with the memory windows set up even before the
 472          * character device.  The userland tool's register i/o and mem read
 473          * will work even in "recovery mode".
 474          */
 475         setup_memwin(sc);
 476         sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
 477             GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
 478         sc->cdev->si_drv1 = sc;
 479
 480         /* Go no further if recovery mode has been requested. */
 481         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 482                 device_printf(dev, "recovery mode.\n");
 483                 goto done;
 484         }
 485
 486         /* Prepare the firmware for operation */
 487         rc = prep_firmware(sc);
 488         if (rc != 0)
 489                 goto done; /* error message displayed already */
 490
 491         rc = get_params__pre_init(sc);
 492         if (rc != 0)
 493                 goto done; /* error message displayed already */
 494
 495         rc = t4_sge_init(sc);
 496         if (rc != 0)
 497                 goto done; /* error message displayed already */
 498
 499         if (sc->flags & MASTER_PF) {
 500                 /* get basic stuff going */
 501                 rc = -t4_fw_initialize(sc, sc->mbox);
 502                 if (rc != 0) {
 503                         device_printf(dev, "early init failed: %d.\n", rc);
 504                         goto done;
 505                 }
 506         }
 507
 508         rc = get_params__post_init(sc);
 509         if (rc != 0)
 510                 goto done; /* error message displayed already */
 511
 512         if (sc->flags & MASTER_PF) {
 513
 514                 /* final tweaks to some settings */
 515
 516                 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
 517                     sc->params.b_wnd);
 518                 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 519                 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 |
 520                     F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0);
 521                 t4_set_reg_field(sc, A_TP_PARA_REG5,
 522                     V_INDICATESIZE(M_INDICATESIZE) |
 523                     F_REARMDDPOFFSET | F_RESETDDPOFFSET,
 524                     V_INDICATESIZE(M_INDICATESIZE) |
 525                     F_REARMDDPOFFSET | F_RESETDDPOFFSET);
 526         } else {
 527                 /*
 528                  * XXX: Verify that we can live with whatever the master driver
 529                  * has done so far, and hope that it doesn't change any global
 530                  * setting from underneath us in the future.
 531                  */
 532         }
 533
 534         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
 535             A_TP_VLAN_PRI_MAP);
 536
 537         for (i = 0; i < NCHAN; i++)
 538                 sc->params.tp.tx_modq[i] = i;
 539
 540         rc = t4_create_dma_tag(sc);
 541         if (rc != 0)
 542                 goto done; /* error message displayed already */
 543
 544         /*
 545          * First pass over all the ports - allocate VIs and initialize some
 546          * basic parameters like mac address, port type, etc.  We also figure
 547          * out whether a port is 10G or 1G and use that information when
 548          * calculating how many interrupts to attempt to allocate.
 549          */
 550         n10g = n1g = 0;
 551         for_each_port(sc, i) {
 552                 struct port_info *pi;
 553
 554                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 555                 sc->port[i] = pi;
 556
 557                 /* These must be set before t4_port_init */
 558                 pi->adapter = sc;
 559                 pi->port_id = i;
 560
 561                 /* Allocate the vi and initialize parameters like mac addr */
 562                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 563                 if (rc != 0) {
 564                         device_printf(dev, "unable to initialize port %d: %d\n",
 565                             i, rc);
 566                         free(pi, M_CXGBE);
 567                         sc->port[i] = NULL;
 568                         goto done;
 569                 }
 570
 571                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 572                     device_get_nameunit(dev), i);
 573                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 574
 575                 if (is_10G_port(pi)) {
 576                         n10g++;
 577                         pi->tmr_idx = t4_tmr_idx_10g;
 578                         pi->pktc_idx = t4_pktc_idx_10g;
 579                 } else {
 580                         n1g++;
 581                         pi->tmr_idx = t4_tmr_idx_1g;
 582                         pi->pktc_idx = t4_pktc_idx_1g;
 583                 }
 584
 585                 pi->xact_addr_filt = -1;
 586
 587                 pi->qsize_rxq = t4_qsize_rxq;
 588                 pi->qsize_txq = t4_qsize_txq;
 589
 590                 pi->dev = device_add_child(dev, "cxgbe", -1);
 591                 if (pi->dev == NULL) {
 592                         device_printf(dev,
 593                             "failed to add device for port %d.\n", i);
 594                         rc = ENXIO;
 595                         goto done;
 596                 }
 597                 device_set_softc(pi->dev, pi);
 598         }
 599
 600         /*
 601          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 602          */
 603         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 604         if (rc != 0)
 605                 goto done; /* error message displayed already */
 606
 607         sc->intr_type = iaq.intr_type;
 608         sc->intr_count = iaq.nirq;
 609         sc->flags |= iaq.intr_flags;
 610
 611         s = &sc->sge;
 612         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 613         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 614         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 615         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 616         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 617
 618 #ifdef TCP_OFFLOAD
 619         if (is_offload(sc)) {
 620
 621                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 622                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 623                 s->neq += s->nofldtxq + s->nofldrxq;
 624                 s->niq += s->nofldrxq;
 625
 626                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 627                     M_CXGBE, M_ZERO | M_WAITOK);
 628                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 629                     M_CXGBE, M_ZERO | M_WAITOK);
 630         }
 631 #endif
 632
 633         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 634             M_ZERO | M_WAITOK);
 635         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 636             M_ZERO | M_WAITOK);
 637         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 638             M_ZERO | M_WAITOK);
 639         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 640             M_ZERO | M_WAITOK);
 641         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 642             M_ZERO | M_WAITOK);
 643
 644         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 645             M_ZERO | M_WAITOK);
 646
 647         t4_init_l2t(sc, M_WAITOK);
 648
 649         /*
 650          * Second pass over the ports.  This time we know the number of rx and
 651          * tx queues that each port should get.
 652          */
 653         rqidx = tqidx = 0;
 654 #ifdef TCP_OFFLOAD
 655         ofld_rqidx = ofld_tqidx = 0;
 656 #endif
 657         for_each_port(sc, i) {
 658                 struct port_info *pi = sc->port[i];
 659
 660                 if (pi == NULL)
 661                         continue;
 662
 663                 pi->first_rxq = rqidx;
 664                 pi->first_txq = tqidx;
 665                 if (is_10G_port(pi)) {
 666                         pi->nrxq = iaq.nrxq10g;
 667                         pi->ntxq = iaq.ntxq10g;
 668                 } else {
 669                         pi->nrxq = iaq.nrxq1g;
 670                         pi->ntxq = iaq.ntxq1g;
 671                 }
 672
 673                 rqidx += pi->nrxq;
 674                 tqidx += pi->ntxq;
 675
 676 #ifdef TCP_OFFLOAD
 677                 if (is_offload(sc)) {
 678                         pi->first_ofld_rxq = ofld_rqidx;
 679                         pi->first_ofld_txq = ofld_tqidx;
 680                         if (is_10G_port(pi)) {
 681                                 pi->nofldrxq = iaq.nofldrxq10g;
 682                                 pi->nofldtxq = iaq.nofldtxq10g;
 683                         } else {
 684                                 pi->nofldrxq = iaq.nofldrxq1g;
 685                                 pi->nofldtxq = iaq.nofldtxq1g;
 686                         }
 687                         ofld_rqidx += pi->nofldrxq;
 688                         ofld_tqidx += pi->nofldtxq;
 689                 }
 690 #endif
 691         }
 692
 693         rc = bus_generic_attach(dev);
 694         if (rc != 0) {
 695                 device_printf(dev,
 696                     "failed to attach all child ports: %d\n", rc);
 697                 goto done;
 698         }
 699
 700         device_printf(dev,
 701             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 702             sc->params.pci.width, sc->params.nports, sc->intr_count,
 703             sc->intr_type == INTR_MSIX ? "MSI-X" :
 704             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 705             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 706
 707         t4_set_desc(sc);
 708
 709 done:
 710         if (rc != 0 && sc->cdev) {
 711                 /* cdev was created and so cxgbetool works; recover that way. */
 712                 device_printf(dev,
 713                     "error during attach, adapter is now in recovery mode.\n");
 714                 rc = 0;
 715         }
 716
 717         if (rc != 0)
 718                 t4_detach(dev);
 719         else
 720                 t4_sysctls(sc);
 721
 722         return (rc);
 723 }
 724
 725 /*
 726  * Idempotent
 727  */
 728 static int
 729 t4_detach(device_t dev)
 730 {
 731         struct adapter *sc;
 732         struct port_info *pi;
 733         int i, rc;
 734
 735         sc = device_get_softc(dev);
 736
 737         if (sc->flags & FULL_INIT_DONE)
 738                 t4_intr_disable(sc);
 739
 740         if (sc->cdev) {
 741                 destroy_dev(sc->cdev);
 742                 sc->cdev = NULL;
 743         }
 744
 745         rc = bus_generic_detach(dev);
 746         if (rc) {
 747                 device_printf(dev,
 748                     "failed to detach child devices: %d\n", rc);
 749                 return (rc);
 750         }
 751
 752         for (i = 0; i < MAX_NPORTS; i++) {
 753                 pi = sc->port[i];
 754                 if (pi) {
 755                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 756                         if (pi->dev)
 757                                 device_delete_child(dev, pi->dev);
 758
 759                         mtx_destroy(&pi->pi_lock);
 760                         free(pi, M_CXGBE);
 761                 }
 762         }
 763
 764         if (sc->flags & FULL_INIT_DONE)
 765                 adapter_full_uninit(sc);
 766
 767         if (sc->flags & FW_OK)
 768                 t4_fw_bye(sc, sc->mbox);
 769
 770         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 771                 pci_release_msi(dev);
 772
 773         if (sc->regs_res)
 774                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 775                     sc->regs_res);
 776
 777         if (sc->msix_res)
 778                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 779                     sc->msix_res);
 780
 781         if (sc->l2t)
 782                 t4_free_l2t(sc->l2t);
 783
 784 #ifdef TCP_OFFLOAD
 785         free(sc->sge.ofld_rxq, M_CXGBE);
 786         free(sc->sge.ofld_txq, M_CXGBE);
 787 #endif
 788         free(sc->irq, M_CXGBE);
 789         free(sc->sge.rxq, M_CXGBE);
 790         free(sc->sge.txq, M_CXGBE);
 791         free(sc->sge.ctrlq, M_CXGBE);
 792         free(sc->sge.iqmap, M_CXGBE);
 793         free(sc->sge.eqmap, M_CXGBE);
 794         free(sc->tids.ftid_tab, M_CXGBE);
 795         t4_destroy_dma_tag(sc);
 796         if (mtx_initialized(&sc->sc_lock)) {
 797                 mtx_lock(&t4_list_lock);
 798                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 799                 mtx_unlock(&t4_list_lock);
 800                 mtx_destroy(&sc->sc_lock);
 801         }
 802
 803         if (mtx_initialized(&sc->sfl_lock))
 804                 mtx_destroy(&sc->sfl_lock);
 805
 806         bzero(sc, sizeof(*sc));
 807
 808         return (0);
 809 }
 810
 811
 812 static int
 813 cxgbe_probe(device_t dev)
 814 {
 815         char buf[128];
 816         struct port_info *pi = device_get_softc(dev);
 817
 818         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 819         device_set_desc_copy(dev, buf);
 820
 821         return (BUS_PROBE_DEFAULT);
 822 }
 823
 824 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 825     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 826     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
 827 #define T4_CAP_ENABLE (T4_CAP)
 828
 829 static int
 830 cxgbe_attach(device_t dev)
 831 {
 832         struct port_info *pi = device_get_softc(dev);
 833         struct ifnet *ifp;
 834
 835         /* Allocate an ifnet and set it up */
 836         ifp = if_alloc(IFT_ETHER);
 837         if (ifp == NULL) {
 838                 device_printf(dev, "Cannot allocate ifnet\n");
 839                 return (ENOMEM);
 840         }
 841         pi->ifp = ifp;
 842         ifp->if_softc = pi;
 843
 844         callout_init(&pi->tick, CALLOUT_MPSAFE);
 845
 846         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 847         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 848
 849         ifp->if_init = cxgbe_init;
 850         ifp->if_ioctl = cxgbe_ioctl;
 851         ifp->if_transmit = cxgbe_transmit;
 852         ifp->if_qflush = cxgbe_qflush;
 853
 854         ifp->if_capabilities = T4_CAP;
 855 #ifdef TCP_OFFLOAD
 856         if (is_offload(pi->adapter))
 857                 ifp->if_capabilities |= IFCAP_TOE4;
 858 #endif
 859         ifp->if_capenable = T4_CAP_ENABLE;
 860         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 861             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 862
 863         /* Initialize ifmedia for this port */
 864         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 865             cxgbe_media_status);
 866         build_medialist(pi);
 867
 868         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 869             EVENTHANDLER_PRI_ANY);
 870
 871         ether_ifattach(ifp, pi->hw_addr);
 872
 873 #ifdef TCP_OFFLOAD
 874         if (is_offload(pi->adapter)) {
 875                 device_printf(dev,
 876                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
 877                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
 878         } else
 879 #endif
 880                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
 881
 882         cxgbe_sysctls(pi);
 883
 884         return (0);
 885 }
 886
 887 static int
 888 cxgbe_detach(device_t dev)
 889 {
 890         struct port_info *pi = device_get_softc(dev);
 891         struct adapter *sc = pi->adapter;
 892         struct ifnet *ifp = pi->ifp;
 893
 894         /* Tell if_ioctl and if_init that the port is going away */
 895         ADAPTER_LOCK(sc);
 896         SET_DOOMED(pi);
 897         wakeup(&sc->flags);
 898         while (IS_BUSY(sc))
 899                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 900         SET_BUSY(sc);
 901         ADAPTER_UNLOCK(sc);
 902
 903         if (pi->vlan_c)
 904                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
 905
 906         PORT_LOCK(pi);
 907         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 908         callout_stop(&pi->tick);
 909         PORT_UNLOCK(pi);
 910         callout_drain(&pi->tick);
 911
 912         /* Let detach proceed even if these fail. */
 913         cxgbe_uninit_synchronized(pi);
 914         port_full_uninit(pi);
 915
 916         ifmedia_removeall(&pi->media);
 917         ether_ifdetach(pi->ifp);
 918         if_free(pi->ifp);
 919
 920         ADAPTER_LOCK(sc);
 921         CLR_BUSY(sc);
 922         wakeup_one(&sc->flags);
 923         ADAPTER_UNLOCK(sc);
 924
 925         return (0);
 926 }
 927
 928 static void
 929 cxgbe_init(void *arg)
 930 {
 931         struct port_info *pi = arg;
 932         struct adapter *sc = pi->adapter;
 933
 934         ADAPTER_LOCK(sc);
 935         cxgbe_init_locked(pi); /* releases adapter lock */
 936         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 937 }
 938
 939 static int
 940 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 941 {
 942         int rc = 0, mtu, flags;
 943         struct port_info *pi = ifp->if_softc;
 944         struct adapter *sc = pi->adapter;
 945         struct ifreq *ifr = (struct ifreq *)data;
 946         uint32_t mask;
 947
 948         switch (cmd) {
 949         case SIOCSIFMTU:
 950                 ADAPTER_LOCK(sc);
 951                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 952                 if (rc) {
 953 fail:
 954                         ADAPTER_UNLOCK(sc);
 955                         return (rc);
 956                 }
 957
 958                 mtu = ifr->ifr_mtu;
 959                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
 960                         rc = EINVAL;
 961                 } else {
 962                         ifp->if_mtu = mtu;
 963                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 964                                 t4_update_fl_bufsize(ifp);
 965                                 PORT_LOCK(pi);
 966                                 rc = update_mac_settings(pi, XGMAC_MTU);
 967                                 PORT_UNLOCK(pi);
 968                         }
 969                 }
 970                 ADAPTER_UNLOCK(sc);
 971                 break;
 972
 973         case SIOCSIFFLAGS:
 974                 ADAPTER_LOCK(sc);
 975                 if (IS_DOOMED(pi)) {
 976                         rc = ENXIO;
 977                         goto fail;
 978                 }
 979                 if (ifp->if_flags & IFF_UP) {
 980                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 981                                 flags = pi->if_flags;
 982                                 if ((ifp->if_flags ^ flags) &
 983                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 984                                         if (IS_BUSY(sc)) {
 985                                                 rc = EBUSY;
 986                                                 goto fail;
 987                                         }
 988                                         PORT_LOCK(pi);
 989                                         rc = update_mac_settings(pi,
 990                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
 991                                         PORT_UNLOCK(pi);
 992                                 }
 993                                 ADAPTER_UNLOCK(sc);
 994                         } else
 995                                 rc = cxgbe_init_locked(pi);
 996                         pi->if_flags = ifp->if_flags;
 997                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 998                         rc = cxgbe_uninit_locked(pi);
 999                 else
1000                         ADAPTER_UNLOCK(sc);
1001
1002                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1003                 break;
1004
1005         case SIOCADDMULTI:
1006         case SIOCDELMULTI: /* these two can be called with a mutex held :-( */
1007                 ADAPTER_LOCK(sc);
1008                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1009                 if (rc)
1010                         goto fail;
1011
1012                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013                         PORT_LOCK(pi);
1014                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1015                         PORT_UNLOCK(pi);
1016                 }
1017                 ADAPTER_UNLOCK(sc);
1018                 break;
1019
1020         case SIOCSIFCAP:
1021                 ADAPTER_LOCK(sc);
1022                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1023                 if (rc)
1024                         goto fail;
1025
1026                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1027                 if (mask & IFCAP_TXCSUM) {
1028                         ifp->if_capenable ^= IFCAP_TXCSUM;
1029                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1030
1031                         if (IFCAP_TSO4 & ifp->if_capenable &&
1032                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1033                                 ifp->if_capenable &= ~IFCAP_TSO4;
1034                                 if_printf(ifp,
1035                                     "tso4 disabled due to -txcsum.\n");
1036                         }
1037                 }
1038                 if (mask & IFCAP_TXCSUM_IPV6) {
1039                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1040                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1041
1042                         if (IFCAP_TSO6 & ifp->if_capenable &&
1043                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1044                                 ifp->if_capenable &= ~IFCAP_TSO6;
1045                                 if_printf(ifp,
1046                                     "tso6 disabled due to -txcsum6.\n");
1047                         }
1048                 }
1049                 if (mask & IFCAP_RXCSUM)
1050                         ifp->if_capenable ^= IFCAP_RXCSUM;
1051                 if (mask & IFCAP_RXCSUM_IPV6)
1052                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1053
1054                 /*
1055                  * Note that we leave CSUM_TSO alone (it is always set).  The
1056                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1057                  * sending a TSO request our way, so it's sufficient to toggle
1058                  * IFCAP_TSOx only.
1059                  */
1060                 if (mask & IFCAP_TSO4) {
1061                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1062                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1063                                 if_printf(ifp, "enable txcsum first.\n");
1064                                 rc = EAGAIN;
1065                                 goto fail;
1066                         }
1067                         ifp->if_capenable ^= IFCAP_TSO4;
1068                 }
1069                 if (mask & IFCAP_TSO6) {
1070                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1071                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1072                                 if_printf(ifp, "enable txcsum6 first.\n");
1073                                 rc = EAGAIN;
1074                                 goto fail;
1075                         }
1076                         ifp->if_capenable ^= IFCAP_TSO6;
1077                 }
1078                 if (mask & IFCAP_LRO) {
1079 #if defined(INET) || defined(INET6)
1080                         int i;
1081                         struct sge_rxq *rxq;
1082
1083                         ifp->if_capenable ^= IFCAP_LRO;
1084                         for_each_rxq(pi, i, rxq) {
1085                                 if (ifp->if_capenable & IFCAP_LRO)
1086                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1087                                 else
1088                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1089                         }
1090 #endif
1091                 }
1092 #ifdef TCP_OFFLOAD
1093                 if (mask & IFCAP_TOE) {
1094                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1095
1096                         rc = toe_capability(pi, enable);
1097                         if (rc != 0)
1098                                 goto fail;
1099
1100                         ifp->if_capenable ^= mask;
1101                 }
1102 #endif
1103                 if (mask & IFCAP_VLAN_HWTAGGING) {
1104                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1105                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1106                                 PORT_LOCK(pi);
1107                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1108                                 PORT_UNLOCK(pi);
1109                         }
1110                 }
1111                 if (mask & IFCAP_VLAN_MTU) {
1112                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1113
1114                         /* Need to find out how to disable auto-mtu-inflation */
1115                 }
1116                 if (mask & IFCAP_VLAN_HWTSO)
1117                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1118                 if (mask & IFCAP_VLAN_HWCSUM)
1119                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1120
1121 #ifdef VLAN_CAPABILITIES
1122                 VLAN_CAPABILITIES(ifp);
1123 #endif
1124                 ADAPTER_UNLOCK(sc);
1125                 break;
1126
1127         case SIOCSIFMEDIA:
1128         case SIOCGIFMEDIA:
1129                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1130                 break;
1131
1132         default:
1133                 rc = ether_ioctl(ifp, cmd, data);
1134         }
1135
1136         return (rc);
1137 }
1138
1139 static int
1140 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1141 {
1142         struct port_info *pi = ifp->if_softc;
1143         struct adapter *sc = pi->adapter;
1144         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1145         struct buf_ring *br;
1146         int rc;
1147
1148         M_ASSERTPKTHDR(m);
1149
1150         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1151                 m_freem(m);
1152                 return (ENETDOWN);
1153         }
1154
1155         if (m->m_flags & M_FLOWID)
1156                 txq += (m->m_pkthdr.flowid % pi->ntxq);
1157         br = txq->br;
1158
1159         if (TXQ_TRYLOCK(txq) == 0) {
1160                 struct sge_eq *eq = &txq->eq;
1161
1162                 /*
1163                  * It is possible that t4_eth_tx finishes up and releases the
1164                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1165                  * need to make sure that this mbuf doesn't just sit there in
1166                  * the drbr.
1167                  */
1168
1169                 rc = drbr_enqueue(ifp, br, m);
1170                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1171                     !(eq->flags & EQ_DOOMED))
1172                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1173                 return (rc);
1174         }
1175
1176         /*
1177          * txq->m is the mbuf that is held up due to a temporary shortage of
1178          * resources and it should be put on the wire first.  Then what's in
1179          * drbr and finally the mbuf that was just passed in to us.
1180          *
1181          * Return code should indicate the fate of the mbuf that was passed in
1182          * this time.
1183          */
1184
1185         TXQ_LOCK_ASSERT_OWNED(txq);
1186         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1187
1188                 /* Queued for transmission. */
1189
1190                 rc = drbr_enqueue(ifp, br, m);
1191                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1192                 (void) t4_eth_tx(ifp, txq, m);
1193                 TXQ_UNLOCK(txq);
1194                 return (rc);
1195         }
1196
1197         /* Direct transmission. */
1198         rc = t4_eth_tx(ifp, txq, m);
1199         if (rc != 0 && txq->m)
1200                 rc = 0; /* held, will be transmitted soon (hopefully) */
1201
1202         TXQ_UNLOCK(txq);
1203         return (rc);
1204 }
1205
1206 static void
1207 cxgbe_qflush(struct ifnet *ifp)
1208 {
1209         struct port_info *pi = ifp->if_softc;
1210         struct sge_txq *txq;
1211         int i;
1212         struct mbuf *m;
1213
1214         /* queues do not exist if !PORT_INIT_DONE. */
1215         if (pi->flags & PORT_INIT_DONE) {
1216                 for_each_txq(pi, i, txq) {
1217                         TXQ_LOCK(txq);
1218                         m_freem(txq->m);
1219                         txq->m = NULL;
1220                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1221                                 m_freem(m);
1222                         TXQ_UNLOCK(txq);
1223                 }
1224         }
1225         if_qflush(ifp);
1226 }
1227
1228 static int
1229 cxgbe_media_change(struct ifnet *ifp)
1230 {
1231         struct port_info *pi = ifp->if_softc;
1232
1233         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1234
1235         return (EOPNOTSUPP);
1236 }
1237
1238 static void
1239 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1240 {
1241         struct port_info *pi = ifp->if_softc;
1242         struct ifmedia_entry *cur = pi->media.ifm_cur;
1243         int speed = pi->link_cfg.speed;
1244         int data = (pi->port_type << 8) | pi->mod_type;
1245
1246         if (cur->ifm_data != data) {
1247                 build_medialist(pi);
1248                 cur = pi->media.ifm_cur;
1249         }
1250
1251         ifmr->ifm_status = IFM_AVALID;
1252         if (!pi->link_cfg.link_ok)
1253                 return;
1254
1255         ifmr->ifm_status |= IFM_ACTIVE;
1256
1257         /* active and current will differ iff current media is autoselect. */
1258         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1259                 return;
1260
1261         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1262         if (speed == SPEED_10000)
1263                 ifmr->ifm_active |= IFM_10G_T;
1264         else if (speed == SPEED_1000)
1265                 ifmr->ifm_active |= IFM_1000_T;
1266         else if (speed == SPEED_100)
1267                 ifmr->ifm_active |= IFM_100_TX;
1268         else if (speed == SPEED_10)
1269                 ifmr->ifm_active |= IFM_10_T;
1270         else
1271                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1272                             speed));
1273 }
1274
1275 void
1276 t4_fatal_err(struct adapter *sc)
1277 {
1278         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1279         t4_intr_disable(sc);
1280         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1281             device_get_nameunit(sc->dev));
1282 }
1283
1284 static int
1285 map_bars(struct adapter *sc)
1286 {
1287         sc->regs_rid = PCIR_BAR(0);
1288         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1289             &sc->regs_rid, RF_ACTIVE);
1290         if (sc->regs_res == NULL) {
1291                 device_printf(sc->dev, "cannot map registers.\n");
1292                 return (ENXIO);
1293         }
1294         sc->bt = rman_get_bustag(sc->regs_res);
1295         sc->bh = rman_get_bushandle(sc->regs_res);
1296         sc->mmio_len = rman_get_size(sc->regs_res);
1297
1298         sc->msix_rid = PCIR_BAR(4);
1299         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1300             &sc->msix_rid, RF_ACTIVE);
1301         if (sc->msix_res == NULL) {
1302                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1303                 return (ENXIO);
1304         }
1305
1306         return (0);
1307 }
1308
1309 static void
1310 setup_memwin(struct adapter *sc)
1311 {
1312         uint32_t bar0;
1313
1314         /*
1315          * Read low 32b of bar0 indirectly via the hardware backdoor mechanism.
1316          * Works from within PCI passthrough environments too, where
1317          * rman_get_start() can return a different value.  We need to program
1318          * the memory window decoders with the actual addresses that will be
1319          * coming across the PCIe link.
1320          */
1321         bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1322         bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1323
1324         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1325                      (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1326                      V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1327
1328         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1329                      (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1330                      V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1331
1332         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1333                      (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1334                      V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1335
1336         /* flush */
1337         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1338 }
1339
1340 static int
1341 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1342     struct intrs_and_queues *iaq)
1343 {
1344         int rc, itype, navail, nrxq10g, nrxq1g, n;
1345         int nofldrxq10g = 0, nofldrxq1g = 0;
1346
1347         bzero(iaq, sizeof(*iaq));
1348
1349         iaq->ntxq10g = t4_ntxq10g;
1350         iaq->ntxq1g = t4_ntxq1g;
1351         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1352         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1353 #ifdef TCP_OFFLOAD
1354         if (is_offload(sc)) {
1355                 iaq->nofldtxq10g = t4_nofldtxq10g;
1356                 iaq->nofldtxq1g = t4_nofldtxq1g;
1357                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1358                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1359         }
1360 #endif
1361
1362         for (itype = INTR_MSIX; itype; itype >>= 1) {
1363
1364                 if ((itype & t4_intr_types) == 0)
1365                         continue;       /* not allowed */
1366
1367                 if (itype == INTR_MSIX)
1368                         navail = pci_msix_count(sc->dev);
1369                 else if (itype == INTR_MSI)
1370                         navail = pci_msi_count(sc->dev);
1371                 else
1372                         navail = 1;
1373 restart:
1374                 if (navail == 0)
1375                         continue;
1376
1377                 iaq->intr_type = itype;
1378                 iaq->intr_flags = 0;
1379
1380                 /*
1381                  * Best option: an interrupt vector for errors, one for the
1382                  * firmware event queue, and one each for each rxq (NIC as well
1383                  * as offload).
1384                  */
1385                 iaq->nirq = T4_EXTRA_INTR;
1386                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1387                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1388                 if (iaq->nirq <= navail &&
1389                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1390                         iaq->intr_flags |= INTR_DIRECT;
1391                         goto allocate;
1392                 }
1393
1394                 /*
1395                  * Second best option: an interrupt vector for errors, one for
1396                  * the firmware event queue, and one each for either NIC or
1397                  * offload rxq's.
1398                  */
1399                 iaq->nirq = T4_EXTRA_INTR;
1400                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1401                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1402                 if (iaq->nirq <= navail &&
1403                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1404                         goto allocate;
1405
1406                 /*
1407                  * Next best option: an interrupt vector for errors, one for the
1408                  * firmware event queue, and at least one per port.  At this
1409                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1410                  * what's available to us.
1411                  */
1412                 iaq->nirq = T4_EXTRA_INTR;
1413                 iaq->nirq += n10g + n1g;
1414                 if (iaq->nirq <= navail) {
1415                         int leftover = navail - iaq->nirq;
1416
1417                         if (n10g > 0) {
1418                                 int target = max(nrxq10g, nofldrxq10g);
1419
1420                                 n = 1;
1421                                 while (n < target && leftover >= n10g) {
1422                                         leftover -= n10g;
1423                                         iaq->nirq += n10g;
1424                                         n++;
1425                                 }
1426                                 iaq->nrxq10g = min(n, nrxq10g);
1427 #ifdef TCP_OFFLOAD
1428                                 if (is_offload(sc))
1429                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1430 #endif
1431                         }
1432
1433                         if (n1g > 0) {
1434                                 int target = max(nrxq1g, nofldrxq1g);
1435
1436                                 n = 1;
1437                                 while (n < target && leftover >= n1g) {
1438                                         leftover -= n1g;
1439                                         iaq->nirq += n1g;
1440                                         n++;
1441                                 }
1442                                 iaq->nrxq1g = min(n, nrxq1g);
1443 #ifdef TCP_OFFLOAD
1444                                 if (is_offload(sc))
1445                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1446 #endif
1447                         }
1448
1449                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1450                                 goto allocate;
1451                 }
1452
1453                 /*
1454                  * Least desirable option: one interrupt vector for everything.
1455                  */
1456                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1457 #ifdef TCP_OFFLOAD
1458                 if (is_offload(sc))
1459                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1460 #endif
1461
1462 allocate:
1463                 navail = iaq->nirq;
1464                 rc = 0;
1465                 if (itype == INTR_MSIX)
1466                         rc = pci_alloc_msix(sc->dev, &navail);
1467                 else if (itype == INTR_MSI)
1468                         rc = pci_alloc_msi(sc->dev, &navail);
1469
1470                 if (rc == 0) {
1471                         if (navail == iaq->nirq)
1472                                 return (0);
1473
1474                         /*
1475                          * Didn't get the number requested.  Use whatever number
1476                          * the kernel is willing to allocate (it's in navail).
1477                          */
1478                         device_printf(sc->dev, "fewer vectors than requested, "
1479                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1480                             itype, iaq->nirq, navail);
1481                         pci_release_msi(sc->dev);
1482                         goto restart;
1483                 }
1484
1485                 device_printf(sc->dev,
1486                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1487                     itype, rc, iaq->nirq, navail);
1488         }
1489
1490         device_printf(sc->dev,
1491             "failed to find a usable interrupt type.  "
1492             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1493             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1494
1495         return (ENXIO);
1496 }
1497
1498 /*
1499  * Install a compatible firmware (if required), establish contact with it (by
1500  * saying hello), and reset the device.  If we end up as the master driver,
1501  * partition adapter resources by providing a configuration file to the
1502  * firmware.
1503  */
1504 static int
1505 prep_firmware(struct adapter *sc)
1506 {
1507         const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1508         int rc;
1509         enum dev_state state;
1510
1511         default_cfg = firmware_get(T4_CFGNAME);
1512
1513         /* Check firmware version and install a different one if necessary */
1514         rc = t4_check_fw_version(sc);
1515         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1516             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1517             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1518             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1519             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1520         if (rc != 0) {
1521                 uint32_t v = 0;
1522
1523                 fw = firmware_get(T4_FWNAME);
1524                 if (fw != NULL) {
1525                         const struct fw_hdr *hdr = (const void *)fw->data;
1526
1527                         v = ntohl(hdr->fw_ver);
1528
1529                         /*
1530                          * The firmware module will not be used if it isn't the
1531                          * same major version as what the driver was compiled
1532                          * with.
1533                          */
1534                         if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
1535                                 device_printf(sc->dev,
1536                                     "Found firmware image but version %d "
1537                                     "can not be used with this driver (%d)\n",
1538                                     G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR);
1539
1540                                 firmware_put(fw, FIRMWARE_UNLOAD);
1541                                 fw = NULL;
1542                         }
1543                 }
1544
1545                 if (fw == NULL && rc < 0) {
1546                         device_printf(sc->dev, "No usable firmware. "
1547                             "card has %d.%d.%d, driver compiled with %d.%d.%d",
1548                             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1549                             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1550                             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1551                             FW_VERSION_MAJOR, FW_VERSION_MINOR,
1552                             FW_VERSION_MICRO);
1553                         rc = EAGAIN;
1554                         goto done;
1555                 }
1556
1557                 /*
1558                  * Always upgrade, even for minor/micro/build mismatches.
1559                  * Downgrade only for a major version mismatch or if
1560                  * force_firmware_install was specified.
1561                  */
1562                 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
1563                         device_printf(sc->dev,
1564                             "installing firmware %d.%d.%d.%d on card.\n",
1565                             G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1566                             G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1567
1568                         rc = -t4_load_fw(sc, fw->data, fw->datasize);
1569                         if (rc != 0) {
1570                                 device_printf(sc->dev,
1571                                     "failed to install firmware: %d\n", rc);
1572                                 goto done;
1573                         } else {
1574                                 /* refresh */
1575                                 (void) t4_check_fw_version(sc);
1576                                 snprintf(sc->fw_version,
1577                                     sizeof(sc->fw_version), "%u.%u.%u.%u",
1578                                     G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1579                                     G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1580                                     G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1581                                     G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1582                         }
1583                 }
1584         }
1585
1586         /* Contact firmware.  */
1587         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1588         if (rc < 0) {
1589                 rc = -rc;
1590                 device_printf(sc->dev,
1591                     "failed to connect to the firmware: %d.\n", rc);
1592                 goto done;
1593         }
1594         if (rc == sc->mbox)
1595                 sc->flags |= MASTER_PF;
1596
1597         /* Reset device */
1598         rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1599         if (rc != 0) {
1600                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1601                 if (rc != ETIMEDOUT && rc != EIO)
1602                         t4_fw_bye(sc, sc->mbox);
1603                 goto done;
1604         }
1605
1606         /* Partition adapter resources as specified in the config file. */
1607         if (sc->flags & MASTER_PF) {
1608                 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
1609                         char s[32];
1610
1611                         snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
1612                         cfg = firmware_get(s);
1613                         if (cfg == NULL) {
1614                                 device_printf(sc->dev,
1615                                     "unable to locate %s module, "
1616                                     "will use default config file.\n", s);
1617                         }
1618                 }
1619
1620                 rc = partition_resources(sc, cfg ? cfg : default_cfg);
1621                 if (rc != 0)
1622                         goto done;      /* error message displayed already */
1623         }
1624
1625         sc->flags |= FW_OK;
1626
1627 done:
1628         if (fw != NULL)
1629                 firmware_put(fw, FIRMWARE_UNLOAD);
1630         if (cfg != NULL)
1631                 firmware_put(cfg, FIRMWARE_UNLOAD);
1632         if (default_cfg != NULL)
1633                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
1634
1635         return (rc);
1636 }
1637
1638 #define FW_PARAM_DEV(param) \
1639         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1640          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1641 #define FW_PARAM_PFVF(param) \
1642         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1643          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1644
1645 /*
1646  * Upload configuration file to card's memory.
1647  */
1648 static int
1649 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1650     uint32_t *ma)
1651 {
1652         int rc, i;
1653         uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1654         const uint32_t *b;
1655
1656         /* Figure out where the firmware wants us to upload it. */
1657         param = FW_PARAM_DEV(CF);
1658         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1659         if (rc != 0) {
1660                 /* Firmwares without config file support will fail this way */
1661                 device_printf(sc->dev,
1662                     "failed to query config file location: %d.\n", rc);
1663                 return (rc);
1664         }
1665         *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1666         *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1667
1668         if (maddr & 3) {
1669                 device_printf(sc->dev,
1670                     "cannot upload config file (type %u, addr %x).\n",
1671                     mtype, maddr);
1672                 return (EFAULT);
1673         }
1674
1675         /* Translate mtype/maddr to an address suitable for the PCIe window */
1676         val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1677         val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1678         switch (mtype) {
1679         case FW_MEMTYPE_CF_EDC0:
1680                 if (!(val & F_EDRAM0_ENABLE))
1681                         goto err;
1682                 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1683                 maddr += G_EDRAM0_BASE(bar) << 20;
1684                 break;
1685
1686         case FW_MEMTYPE_CF_EDC1:
1687                 if (!(val & F_EDRAM1_ENABLE))
1688                         goto err;
1689                 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1690                 maddr += G_EDRAM1_BASE(bar) << 20;
1691                 break;
1692
1693         case FW_MEMTYPE_CF_EXTMEM:
1694                 if (!(val & F_EXT_MEM_ENABLE))
1695                         goto err;
1696                 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1697                 maddr += G_EXT_MEM_BASE(bar) << 20;
1698                 break;
1699
1700         default:
1701 err:
1702                 device_printf(sc->dev,
1703                     "cannot upload config file (type %u, enabled %u).\n",
1704                     mtype, val);
1705                 return (EFAULT);
1706         }
1707
1708         /*
1709          * Position the PCIe window (we use memwin2) to the 16B aligned area
1710          * just at/before the upload location.
1711          */
1712         win = maddr & ~0xf;
1713         off = maddr - win;  /* offset from the start of the window. */
1714         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1715         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1716
1717         remaining = fw->datasize;
1718         if (remaining > FLASH_CFG_MAX_SIZE ||
1719             remaining > MEMWIN2_APERTURE - off) {
1720                 device_printf(sc->dev, "cannot upload config file all at once "
1721                     "(size %u, max %u, room %u).\n",
1722                     remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1723                 return (EFBIG);
1724         }
1725
1726         /*
1727          * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1728          * stuffing/comments at the end of the config file so it's ok to simply
1729          * throw away the last remaining bytes when the config file is not an
1730          * exact multiple of 4.
1731          */
1732         b = fw->data;
1733         for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1734                 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1735
1736         return (rc);
1737 }
1738
1739 /*
1740  * Partition chip resources for use between various PFs, VFs, etc.  This is done
1741  * by uploading the firmware configuration file to the adapter and instructing
1742  * the firmware to process it.
1743  */
1744 static int
1745 partition_resources(struct adapter *sc, const struct firmware *cfg)
1746 {
1747         int rc;
1748         struct fw_caps_config_cmd caps;
1749         uint32_t mtype, maddr, finicsum, cfcsum;
1750
1751         rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1752         if (rc != 0) {
1753                 mtype = FW_MEMTYPE_CF_FLASH;
1754                 maddr = t4_flash_cfg_addr(sc);
1755         }
1756
1757         bzero(&caps, sizeof(caps));
1758         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1759             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1760         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1761             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1762             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1763         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1764         if (rc != 0) {
1765                 device_printf(sc->dev,
1766                     "failed to pre-process config file: %d.\n", rc);
1767                 return (rc);
1768         }
1769
1770         finicsum = be32toh(caps.finicsum);
1771         cfcsum = be32toh(caps.cfcsum);
1772         if (finicsum != cfcsum) {
1773                 device_printf(sc->dev,
1774                     "WARNING: config file checksum mismatch: %08x %08x\n",
1775                     finicsum, cfcsum);
1776         }
1777         sc->cfcsum = cfcsum;
1778
1779 #define LIMIT_CAPS(x) do { \
1780         caps.x &= htobe16(t4_##x##_allowed); \
1781         sc->x = htobe16(caps.x); \
1782 } while (0)
1783
1784         /*
1785          * Let the firmware know what features will (not) be used so it can tune
1786          * things accordingly.
1787          */
1788         LIMIT_CAPS(linkcaps);
1789         LIMIT_CAPS(niccaps);
1790         LIMIT_CAPS(toecaps);
1791         LIMIT_CAPS(rdmacaps);
1792         LIMIT_CAPS(iscsicaps);
1793         LIMIT_CAPS(fcoecaps);
1794 #undef LIMIT_CAPS
1795
1796         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1797             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1798         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1799         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1800         if (rc != 0) {
1801                 device_printf(sc->dev,
1802                     "failed to process config file: %d.\n", rc);
1803                 return (rc);
1804         }
1805
1806         return (0);
1807 }
1808
1809 /*
1810  * Retrieve parameters that are needed (or nice to have) prior to calling
1811  * t4_sge_init and t4_fw_initialize.
1812  */
1813 static int
1814 get_params__pre_init(struct adapter *sc)
1815 {
1816         int rc;
1817         uint32_t param[2], val[2];
1818         struct fw_devlog_cmd cmd;
1819         struct devlog_params *dlog = &sc->params.devlog;
1820
1821         param[0] = FW_PARAM_DEV(PORTVEC);
1822         param[1] = FW_PARAM_DEV(CCLK);
1823         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1824         if (rc != 0) {
1825                 device_printf(sc->dev,
1826                     "failed to query parameters (pre_init): %d.\n", rc);
1827                 return (rc);
1828         }
1829
1830         sc->params.portvec = val[0];
1831         sc->params.nports = 0;
1832         while (val[0]) {
1833                 sc->params.nports++;
1834                 val[0] &= val[0] - 1;
1835         }
1836
1837         sc->params.vpd.cclk = val[1];
1838
1839         /* Read device log parameters. */
1840         bzero(&cmd, sizeof(cmd));
1841         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1842             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1843         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1844         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1845         if (rc != 0) {
1846                 device_printf(sc->dev,
1847                     "failed to get devlog parameters: %d.\n", rc);
1848                 bzero(dlog, sizeof (*dlog));
1849                 rc = 0; /* devlog isn't critical for device operation */
1850         } else {
1851                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1852                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1853                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1854                 dlog->size = be32toh(cmd.memsize_devlog);
1855         }
1856
1857         return (rc);
1858 }
1859
1860 /*
1861  * Retrieve various parameters that are of interest to the driver.  The device
1862  * has been initialized by the firmware at this point.
1863  */
1864 static int
1865 get_params__post_init(struct adapter *sc)
1866 {
1867         int rc;
1868         uint32_t param[7], val[7];
1869         struct fw_caps_config_cmd caps;
1870
1871         param[0] = FW_PARAM_PFVF(IQFLINT_START);
1872         param[1] = FW_PARAM_PFVF(EQ_START);
1873         param[2] = FW_PARAM_PFVF(FILTER_START);
1874         param[3] = FW_PARAM_PFVF(FILTER_END);
1875         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
1876         if (rc != 0) {
1877                 device_printf(sc->dev,
1878                     "failed to query parameters (post_init): %d.\n", rc);
1879                 return (rc);
1880         }
1881
1882         sc->sge.iq_start = val[0];
1883         sc->sge.eq_start = val[1];
1884         sc->tids.ftid_base = val[2];
1885         sc->tids.nftids = val[3] - val[2] + 1;
1886
1887         /* get capabilites */
1888         bzero(&caps, sizeof(caps));
1889         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1890             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1891         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1892         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1893         if (rc != 0) {
1894                 device_printf(sc->dev,
1895                     "failed to get card capabilities: %d.\n", rc);
1896                 return (rc);
1897         }
1898
1899         if (caps.toecaps) {
1900                 /* query offload-related parameters */
1901                 param[0] = FW_PARAM_DEV(NTID);
1902                 param[1] = FW_PARAM_PFVF(SERVER_START);
1903                 param[2] = FW_PARAM_PFVF(SERVER_END);
1904                 param[3] = FW_PARAM_PFVF(TDDP_START);
1905                 param[4] = FW_PARAM_PFVF(TDDP_END);
1906                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1907                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1908                 if (rc != 0) {
1909                         device_printf(sc->dev,
1910                             "failed to query TOE parameters: %d.\n", rc);
1911                         return (rc);
1912                 }
1913                 sc->tids.ntids = val[0];
1914                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1915                 sc->tids.stid_base = val[1];
1916                 sc->tids.nstids = val[2] - val[1] + 1;
1917                 sc->vres.ddp.start = val[3];
1918                 sc->vres.ddp.size = val[4] - val[3] + 1;
1919                 sc->params.ofldq_wr_cred = val[5];
1920                 sc->params.offload = 1;
1921         }
1922         if (caps.rdmacaps) {
1923                 param[0] = FW_PARAM_PFVF(STAG_START);
1924                 param[1] = FW_PARAM_PFVF(STAG_END);
1925                 param[2] = FW_PARAM_PFVF(RQ_START);
1926                 param[3] = FW_PARAM_PFVF(RQ_END);
1927                 param[4] = FW_PARAM_PFVF(PBL_START);
1928                 param[5] = FW_PARAM_PFVF(PBL_END);
1929                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1930                 if (rc != 0) {
1931                         device_printf(sc->dev,
1932                             "failed to query RDMA parameters(1): %d.\n", rc);
1933                         return (rc);
1934                 }
1935                 sc->vres.stag.start = val[0];
1936                 sc->vres.stag.size = val[1] - val[0] + 1;
1937                 sc->vres.rq.start = val[2];
1938                 sc->vres.rq.size = val[3] - val[2] + 1;
1939                 sc->vres.pbl.start = val[4];
1940                 sc->vres.pbl.size = val[5] - val[4] + 1;
1941
1942                 param[0] = FW_PARAM_PFVF(SQRQ_START);
1943                 param[1] = FW_PARAM_PFVF(SQRQ_END);
1944                 param[2] = FW_PARAM_PFVF(CQ_START);
1945                 param[3] = FW_PARAM_PFVF(CQ_END);
1946                 param[4] = FW_PARAM_PFVF(OCQ_START);
1947                 param[5] = FW_PARAM_PFVF(OCQ_END);
1948                 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
1949                 if (rc != 0) {
1950                         device_printf(sc->dev,
1951                             "failed to query RDMA parameters(2): %d.\n", rc);
1952                         return (rc);
1953                 }
1954                 sc->vres.qp.start = val[0];
1955                 sc->vres.qp.size = val[1] - val[0] + 1;
1956                 sc->vres.cq.start = val[2];
1957                 sc->vres.cq.size = val[3] - val[2] + 1;
1958                 sc->vres.ocq.start = val[4];
1959                 sc->vres.ocq.size = val[5] - val[4] + 1;
1960         }
1961         if (caps.iscsicaps) {
1962                 param[0] = FW_PARAM_PFVF(ISCSI_START);
1963                 param[1] = FW_PARAM_PFVF(ISCSI_END);
1964                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1965                 if (rc != 0) {
1966                         device_printf(sc->dev,
1967                             "failed to query iSCSI parameters: %d.\n", rc);
1968                         return (rc);
1969                 }
1970                 sc->vres.iscsi.start = val[0];
1971                 sc->vres.iscsi.size = val[1] - val[0] + 1;
1972         }
1973
1974         /* These are finalized by FW initialization, load their values now */
1975         val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1976         sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1977         sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1978         t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1979
1980         return (rc);
1981 }
1982
1983 #undef FW_PARAM_PFVF
1984 #undef FW_PARAM_DEV
1985
1986 static void
1987 t4_set_desc(struct adapter *sc)
1988 {
1989         char buf[128];
1990         struct adapter_params *p = &sc->params;
1991
1992         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
1993             p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
1994
1995         device_set_desc_copy(sc->dev, buf);
1996 }
1997
1998 static void
1999 build_medialist(struct port_info *pi)
2000 {
2001         struct ifmedia *media = &pi->media;
2002         int data, m;
2003
2004         PORT_LOCK(pi);
2005
2006         ifmedia_removeall(media);
2007
2008         m = IFM_ETHER | IFM_FDX;
2009         data = (pi->port_type << 8) | pi->mod_type;
2010
2011         switch(pi->port_type) {
2012         case FW_PORT_TYPE_BT_XFI:
2013                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2014                 break;
2015
2016         case FW_PORT_TYPE_BT_XAUI:
2017                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2018                 /* fall through */
2019
2020         case FW_PORT_TYPE_BT_SGMII:
2021                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2022                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2023                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2024                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2025                 break;
2026
2027         case FW_PORT_TYPE_CX4:
2028                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2029                 ifmedia_set(media, m | IFM_10G_CX4);
2030                 break;
2031
2032         case FW_PORT_TYPE_SFP:
2033         case FW_PORT_TYPE_FIBER_XFI:
2034         case FW_PORT_TYPE_FIBER_XAUI:
2035                 switch (pi->mod_type) {
2036
2037                 case FW_PORT_MOD_TYPE_LR:
2038                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2039                         ifmedia_set(media, m | IFM_10G_LR);
2040                         break;
2041
2042                 case FW_PORT_MOD_TYPE_SR:
2043                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2044                         ifmedia_set(media, m | IFM_10G_SR);
2045                         break;
2046
2047                 case FW_PORT_MOD_TYPE_LRM:
2048                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2049                         ifmedia_set(media, m | IFM_10G_LRM);
2050                         break;
2051
2052                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2053                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2054                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2055                         ifmedia_set(media, m | IFM_10G_TWINAX);
2056                         break;
2057
2058                 case FW_PORT_MOD_TYPE_NONE:
2059                         m &= ~IFM_FDX;
2060                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2061                         ifmedia_set(media, m | IFM_NONE);
2062                         break;
2063
2064                 case FW_PORT_MOD_TYPE_NA:
2065                 case FW_PORT_MOD_TYPE_ER:
2066                 default:
2067                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2068                         ifmedia_set(media, m | IFM_UNKNOWN);
2069                         break;
2070                 }
2071                 break;
2072
2073         case FW_PORT_TYPE_KX4:
2074         case FW_PORT_TYPE_KX:
2075         case FW_PORT_TYPE_KR:
2076         default:
2077                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2078                 ifmedia_set(media, m | IFM_UNKNOWN);
2079                 break;
2080         }
2081
2082         PORT_UNLOCK(pi);
2083 }
2084
2085 #define FW_MAC_EXACT_CHUNK      7
2086
2087 /*
2088  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2089  * indicates which parameters should be programmed (the rest are left alone).
2090  */
2091 static int
2092 update_mac_settings(struct port_info *pi, int flags)
2093 {
2094         int rc;
2095         struct ifnet *ifp = pi->ifp;
2096         struct adapter *sc = pi->adapter;
2097         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2098
2099         PORT_LOCK_ASSERT_OWNED(pi);
2100         KASSERT(flags, ("%s: not told what to update.", __func__));
2101
2102         if (flags & XGMAC_MTU)
2103                 mtu = ifp->if_mtu;
2104
2105         if (flags & XGMAC_PROMISC)
2106                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2107
2108         if (flags & XGMAC_ALLMULTI)
2109                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2110
2111         if (flags & XGMAC_VLANEX)
2112                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2113
2114         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2115             vlanex, false);
2116         if (rc) {
2117                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2118                 return (rc);
2119         }
2120
2121         if (flags & XGMAC_UCADDR) {
2122                 uint8_t ucaddr[ETHER_ADDR_LEN];
2123
2124                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2125                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2126                     ucaddr, true, true);
2127                 if (rc < 0) {
2128                         rc = -rc;
2129                         if_printf(ifp, "change_mac failed: %d\n", rc);
2130                         return (rc);
2131                 } else {
2132                         pi->xact_addr_filt = rc;
2133                         rc = 0;
2134                 }
2135         }
2136
2137         if (flags & XGMAC_MCADDRS) {
2138                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2139                 int del = 1;
2140                 uint64_t hash = 0;
2141                 struct ifmultiaddr *ifma;
2142                 int i = 0, j;
2143
2144                 if_maddr_rlock(ifp);
2145                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2146                         if (ifma->ifma_addr->sa_family != AF_LINK)
2147                                 continue;
2148                         mcaddr[i++] =
2149                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2150
2151                         if (i == FW_MAC_EXACT_CHUNK) {
2152                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2153                                     del, i, mcaddr, NULL, &hash, 0);
2154                                 if (rc < 0) {
2155                                         rc = -rc;
2156                                         for (j = 0; j < i; j++) {
2157                                                 if_printf(ifp,
2158                                                     "failed to add mc address"
2159                                                     " %02x:%02x:%02x:"
2160                                                     "%02x:%02x:%02x rc=%d\n",
2161                                                     mcaddr[j][0], mcaddr[j][1],
2162                                                     mcaddr[j][2], mcaddr[j][3],
2163                                                     mcaddr[j][4], mcaddr[j][5],
2164                                                     rc);
2165                                         }
2166                                         goto mcfail;
2167                                 }
2168                                 del = 0;
2169                                 i = 0;
2170                         }
2171                 }
2172                 if (i > 0) {
2173                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2174                             del, i, mcaddr, NULL, &hash, 0);
2175                         if (rc < 0) {
2176                                 rc = -rc;
2177                                 for (j = 0; j < i; j++) {
2178                                         if_printf(ifp,
2179                                             "failed to add mc address"
2180                                             " %02x:%02x:%02x:"
2181                                             "%02x:%02x:%02x rc=%d\n",
2182                                             mcaddr[j][0], mcaddr[j][1],
2183                                             mcaddr[j][2], mcaddr[j][3],
2184                                             mcaddr[j][4], mcaddr[j][5],
2185                                             rc);
2186                                 }
2187                                 goto mcfail;
2188                         }
2189                 }
2190
2191                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2192                 if (rc != 0)
2193                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2194 mcfail:
2195                 if_maddr_runlock(ifp);
2196         }
2197
2198         return (rc);
2199 }
2200
2201 static int
2202 cxgbe_init_locked(struct port_info *pi)
2203 {
2204         struct adapter *sc = pi->adapter;
2205         int rc = 0;
2206
2207         ADAPTER_LOCK_ASSERT_OWNED(sc);
2208
2209         while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2210                 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) {
2211                         rc = EINTR;
2212                         goto done;
2213                 }
2214         }
2215         if (IS_DOOMED(pi)) {
2216                 rc = ENXIO;
2217                 goto done;
2218         }
2219         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2220
2221         /* Give up the adapter lock, port init code can sleep. */
2222         SET_BUSY(sc);
2223         ADAPTER_UNLOCK(sc);
2224
2225         rc = cxgbe_init_synchronized(pi);
2226
2227 done:
2228         ADAPTER_LOCK(sc);
2229         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2230         CLR_BUSY(sc);
2231         wakeup_one(&sc->flags);
2232         ADAPTER_UNLOCK(sc);
2233         return (rc);
2234 }
2235
2236 static int
2237 cxgbe_init_synchronized(struct port_info *pi)
2238 {
2239         struct adapter *sc = pi->adapter;
2240         struct ifnet *ifp = pi->ifp;
2241         int rc = 0;
2242
2243         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2244
2245         if (isset(&sc->open_device_map, pi->port_id)) {
2246                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2247                     ("mismatch between open_device_map and if_drv_flags"));
2248                 return (0);     /* already running */
2249         }
2250
2251         if (!(sc->flags & FULL_INIT_DONE) &&
2252             ((rc = adapter_full_init(sc)) != 0))
2253                 return (rc);    /* error message displayed already */
2254
2255         if (!(pi->flags & PORT_INIT_DONE) &&
2256             ((rc = port_full_init(pi)) != 0))
2257                 return (rc); /* error message displayed already */
2258
2259         PORT_LOCK(pi);
2260         rc = update_mac_settings(pi, XGMAC_ALL);
2261         PORT_UNLOCK(pi);
2262         if (rc)
2263                 goto done;      /* error message displayed already */
2264
2265         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2266         if (rc != 0) {
2267                 if_printf(ifp, "start_link failed: %d\n", rc);
2268                 goto done;
2269         }
2270
2271         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2272         if (rc != 0) {
2273                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2274                 goto done;
2275         }
2276
2277         /* all ok */
2278         setbit(&sc->open_device_map, pi->port_id);
2279         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2280
2281         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2282 done:
2283         if (rc != 0)
2284                 cxgbe_uninit_synchronized(pi);
2285
2286         return (rc);
2287 }
2288
2289 static int
2290 cxgbe_uninit_locked(struct port_info *pi)
2291 {
2292         struct adapter *sc = pi->adapter;
2293         int rc;
2294
2295         ADAPTER_LOCK_ASSERT_OWNED(sc);
2296
2297         while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2298                 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) {
2299                         rc = EINTR;
2300                         goto done;
2301                 }
2302         }
2303         if (IS_DOOMED(pi)) {
2304                 rc = ENXIO;
2305                 goto done;
2306         }
2307         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2308         SET_BUSY(sc);
2309         ADAPTER_UNLOCK(sc);
2310
2311         rc = cxgbe_uninit_synchronized(pi);
2312
2313         ADAPTER_LOCK(sc);
2314         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2315         CLR_BUSY(sc);
2316         wakeup_one(&sc->flags);
2317 done:
2318         ADAPTER_UNLOCK(sc);
2319         return (rc);
2320 }
2321
2322 /*
2323  * Idempotent.
2324  */
2325 static int
2326 cxgbe_uninit_synchronized(struct port_info *pi)
2327 {
2328         struct adapter *sc = pi->adapter;
2329         struct ifnet *ifp = pi->ifp;
2330         int rc;
2331
2332         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2333
2334         /*
2335          * Disable the VI so that all its data in either direction is discarded
2336          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2337          * tick) intact as the TP can deliver negative advice or data that it's
2338          * holding in its RAM (for an offloaded connection) even after the VI is
2339          * disabled.
2340          */
2341         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2342         if (rc) {
2343                 if_printf(ifp, "disable_vi failed: %d\n", rc);
2344                 return (rc);
2345         }
2346
2347         clrbit(&sc->open_device_map, pi->port_id);
2348         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2349
2350         pi->link_cfg.link_ok = 0;
2351         pi->link_cfg.speed = 0;
2352         t4_os_link_changed(sc, pi->port_id, 0);
2353
2354         return (0);
2355 }
2356
2357 #define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \
2358         rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \
2359         if (rc != 0) \
2360                 goto done; \
2361 } while (0)
2362
2363 static int
2364 adapter_full_init(struct adapter *sc)
2365 {
2366         int rc, i, rid, p, q;
2367         char s[8];
2368         struct irq *irq;
2369         struct port_info *pi;
2370         struct sge_rxq *rxq;
2371 #ifdef TCP_OFFLOAD
2372         struct sge_ofld_rxq *ofld_rxq;
2373 #endif
2374
2375         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2376         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2377             ("%s: FULL_INIT_DONE already", __func__));
2378
2379         /*
2380          * queues that belong to the adapter (not any particular port).
2381          */
2382         rc = t4_setup_adapter_queues(sc);
2383         if (rc != 0)
2384                 goto done;
2385
2386         for (i = 0; i < ARRAY_SIZE(sc->tq); i++) {
2387                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2388                     taskqueue_thread_enqueue, &sc->tq[i]);
2389                 if (sc->tq[i] == NULL) {
2390                         device_printf(sc->dev,
2391                             "failed to allocate task queue %d\n", i);
2392                         rc = ENOMEM;
2393                         goto done;
2394                 }
2395                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2396                     device_get_nameunit(sc->dev), i);
2397         }
2398
2399         /*
2400          * Setup interrupts.
2401          */
2402         irq = &sc->irq[0];
2403         rid = sc->intr_type == INTR_INTX ? 0 : 1;
2404         if (sc->intr_count == 1) {
2405                 KASSERT(!(sc->flags & INTR_DIRECT),
2406                     ("%s: single interrupt && INTR_DIRECT?", __func__));
2407
2408                 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all");
2409         } else {
2410                 /* Multiple interrupts. */
2411                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2412                     ("%s: too few intr.", __func__));
2413
2414                 /* The first one is always error intr */
2415                 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err");
2416                 irq++;
2417                 rid++;
2418
2419                 /* The second one is always the firmware event queue */
2420                 T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
2421                 irq++;
2422                 rid++;
2423
2424                 /*
2425                  * Note that if INTR_DIRECT is not set then either the NIC rx
2426                  * queues or (exclusive or) the TOE rx queueus will be taking
2427                  * direct interrupts.
2428                  *
2429                  * There is no need to check for is_offload(sc) as nofldrxq
2430                  * will be 0 if offload is disabled.
2431                  */
2432                 for_each_port(sc, p) {
2433                         pi = sc->port[p];
2434
2435 #ifdef TCP_OFFLOAD
2436                         /*
2437                          * Skip over the NIC queues if they aren't taking direct
2438                          * interrupts.
2439                          */
2440                         if (!(sc->flags & INTR_DIRECT) &&
2441                             pi->nofldrxq > pi->nrxq)
2442                                 goto ofld_queues;
2443 #endif
2444                         rxq = &sc->sge.rxq[pi->first_rxq];
2445                         for (q = 0; q < pi->nrxq; q++, rxq++) {
2446                                 snprintf(s, sizeof(s), "%d.%d", p, q);
2447                                 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s);
2448                                 irq++;
2449                                 rid++;
2450                         }
2451
2452 #ifdef TCP_OFFLOAD
2453                         /*
2454                          * Skip over the offload queues if they aren't taking
2455                          * direct interrupts.
2456                          */
2457                         if (!(sc->flags & INTR_DIRECT))
2458                                 continue;
2459 ofld_queues:
2460                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2461                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2462                                 snprintf(s, sizeof(s), "%d,%d", p, q);
2463                                 T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s);
2464                                 irq++;
2465                                 rid++;
2466                         }
2467 #endif
2468                 }
2469         }
2470
2471         t4_intr_enable(sc);
2472         sc->flags |= FULL_INIT_DONE;
2473 done:
2474         if (rc != 0)
2475                 adapter_full_uninit(sc);
2476
2477         return (rc);
2478 }
2479 #undef T4_ALLOC_IRQ
2480
2481 static int
2482 adapter_full_uninit(struct adapter *sc)
2483 {
2484         int i;
2485
2486         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2487
2488         t4_teardown_adapter_queues(sc);
2489
2490         for (i = 0; i < sc->intr_count; i++)
2491                 t4_free_irq(sc, &sc->irq[i]);
2492
2493         for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) {
2494                 taskqueue_free(sc->tq[i]);
2495                 sc->tq[i] = NULL;
2496         }
2497
2498         sc->flags &= ~FULL_INIT_DONE;
2499
2500         return (0);
2501 }
2502
2503 static int
2504 port_full_init(struct port_info *pi)
2505 {
2506         struct adapter *sc = pi->adapter;
2507         struct ifnet *ifp = pi->ifp;
2508         uint16_t *rss;
2509         struct sge_rxq *rxq;
2510         int rc, i;
2511
2512         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2513         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2514             ("%s: PORT_INIT_DONE already", __func__));
2515
2516         sysctl_ctx_init(&pi->ctx);
2517         pi->flags |= PORT_SYSCTL_CTX;
2518
2519         /*
2520          * Allocate tx/rx/fl queues for this port.
2521          */
2522         rc = t4_setup_port_queues(pi);
2523         if (rc != 0)
2524                 goto done;      /* error message displayed already */
2525
2526         /*
2527          * Setup RSS for this port.
2528          */
2529         rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2530             M_ZERO | M_WAITOK);
2531         for_each_rxq(pi, i, rxq) {
2532                 rss[i] = rxq->iq.abs_id;
2533         }
2534         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2535             pi->rss_size, rss, pi->nrxq);
2536         free(rss, M_CXGBE);
2537         if (rc != 0) {
2538                 if_printf(ifp, "rss_config failed: %d\n", rc);
2539                 goto done;
2540         }
2541
2542         pi->flags |= PORT_INIT_DONE;
2543 done:
2544         if (rc != 0)
2545                 port_full_uninit(pi);
2546
2547         return (rc);
2548 }
2549
2550 /*
2551  * Idempotent.
2552  */
2553 static int
2554 port_full_uninit(struct port_info *pi)
2555 {
2556         struct adapter *sc = pi->adapter;
2557         int i;
2558         struct sge_rxq *rxq;
2559         struct sge_txq *txq;
2560 #ifdef TCP_OFFLOAD
2561         struct sge_ofld_rxq *ofld_rxq;
2562         struct sge_wrq *ofld_txq;
2563 #endif
2564
2565         if (pi->flags & PORT_INIT_DONE) {
2566
2567                 /* Need to quiesce queues.  XXX: ctrl queues? */
2568
2569                 for_each_txq(pi, i, txq) {
2570                         quiesce_eq(sc, &txq->eq);
2571                 }
2572
2573 #ifdef TCP_OFFLOAD
2574                 for_each_ofld_txq(pi, i, ofld_txq) {
2575                         quiesce_eq(sc, &ofld_txq->eq);
2576                 }
2577 #endif
2578
2579                 for_each_rxq(pi, i, rxq) {
2580                         quiesce_iq(sc, &rxq->iq);
2581                         quiesce_fl(sc, &rxq->fl);
2582                 }
2583
2584 #ifdef TCP_OFFLOAD
2585                 for_each_ofld_rxq(pi, i, ofld_rxq) {
2586                         quiesce_iq(sc, &ofld_rxq->iq);
2587                         quiesce_fl(sc, &ofld_rxq->fl);
2588                 }
2589 #endif
2590         }
2591
2592         t4_teardown_port_queues(pi);
2593         pi->flags &= ~PORT_INIT_DONE;
2594
2595         return (0);
2596 }
2597
2598 static void
2599 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
2600 {
2601         EQ_LOCK(eq);
2602         eq->flags |= EQ_DOOMED;
2603
2604         /*
2605          * Wait for the response to a credit flush if one's
2606          * pending.
2607          */
2608         while (eq->flags & EQ_CRFLUSHED)
2609                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2610         EQ_UNLOCK(eq);
2611
2612         callout_drain(&eq->tx_callout); /* XXX: iffy */
2613         pause("callout", 10);           /* Still iffy */
2614
2615         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2616 }
2617
2618 static void
2619 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
2620 {
2621         (void) sc;      /* unused */
2622
2623         /* Synchronize with the interrupt handler */
2624         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2625                 pause("iqfree", 1);
2626 }
2627
2628 static void
2629 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
2630 {
2631         mtx_lock(&sc->sfl_lock);
2632         FL_LOCK(fl);
2633         fl->flags |= FL_DOOMED;
2634         FL_UNLOCK(fl);
2635         mtx_unlock(&sc->sfl_lock);
2636
2637         callout_drain(&sc->sfl_callout);
2638         KASSERT((fl->flags & FL_STARVING) == 0,
2639             ("%s: still starving", __func__));
2640 }
2641
2642 static int
2643 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2644     driver_intr_t *handler, void *arg, char *name)
2645 {
2646         int rc;
2647
2648         irq->rid = rid;
2649         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2650             RF_SHAREABLE | RF_ACTIVE);
2651         if (irq->res == NULL) {
2652                 device_printf(sc->dev,
2653                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2654                 return (ENOMEM);
2655         }
2656
2657         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2658             NULL, handler, arg, &irq->tag);
2659         if (rc != 0) {
2660                 device_printf(sc->dev,
2661                     "failed to setup interrupt for rid %d, name %s: %d\n",
2662                     rid, name, rc);
2663         } else if (name)
2664                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2665
2666         return (rc);
2667 }
2668
2669 static int
2670 t4_free_irq(struct adapter *sc, struct irq *irq)
2671 {
2672         if (irq->tag)
2673                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
2674         if (irq->res)
2675                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2676
2677         bzero(irq, sizeof(*irq));
2678
2679         return (0);
2680 }
2681
2682 static void
2683 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2684     unsigned int end)
2685 {
2686         uint32_t *p = (uint32_t *)(buf + start);
2687
2688         for ( ; start <= end; start += sizeof(uint32_t))
2689                 *p++ = t4_read_reg(sc, start);
2690 }
2691
2692 static void
2693 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2694 {
2695         int i;
2696         static const unsigned int reg_ranges[] = {
2697                 0x1008, 0x1108,
2698                 0x1180, 0x11b4,
2699                 0x11fc, 0x123c,
2700                 0x1300, 0x173c,
2701                 0x1800, 0x18fc,
2702                 0x3000, 0x30d8,
2703                 0x30e0, 0x5924,
2704                 0x5960, 0x59d4,
2705                 0x5a00, 0x5af8,
2706                 0x6000, 0x6098,
2707                 0x6100, 0x6150,
2708                 0x6200, 0x6208,
2709                 0x6240, 0x6248,
2710                 0x6280, 0x6338,
2711                 0x6370, 0x638c,
2712                 0x6400, 0x643c,
2713                 0x6500, 0x6524,
2714                 0x6a00, 0x6a38,
2715                 0x6a60, 0x6a78,
2716                 0x6b00, 0x6b84,
2717                 0x6bf0, 0x6c84,
2718                 0x6cf0, 0x6d84,
2719                 0x6df0, 0x6e84,
2720                 0x6ef0, 0x6f84,
2721                 0x6ff0, 0x7084,
2722                 0x70f0, 0x7184,
2723                 0x71f0, 0x7284,
2724                 0x72f0, 0x7384,
2725                 0x73f0, 0x7450,
2726                 0x7500, 0x7530,
2727                 0x7600, 0x761c,
2728                 0x7680, 0x76cc,
2729                 0x7700, 0x7798,
2730                 0x77c0, 0x77fc,
2731                 0x7900, 0x79fc,
2732                 0x7b00, 0x7c38,
2733                 0x7d00, 0x7efc,
2734                 0x8dc0, 0x8e1c,
2735                 0x8e30, 0x8e78,
2736                 0x8ea0, 0x8f6c,
2737                 0x8fc0, 0x9074,
2738                 0x90fc, 0x90fc,
2739                 0x9400, 0x9458,
2740                 0x9600, 0x96bc,
2741                 0x9800, 0x9808,
2742                 0x9820, 0x983c,
2743                 0x9850, 0x9864,
2744                 0x9c00, 0x9c6c,
2745                 0x9c80, 0x9cec,
2746                 0x9d00, 0x9d6c,
2747                 0x9d80, 0x9dec,
2748                 0x9e00, 0x9e6c,
2749                 0x9e80, 0x9eec,
2750                 0x9f00, 0x9f6c,
2751                 0x9f80, 0x9fec,
2752                 0xd004, 0xd03c,
2753                 0xdfc0, 0xdfe0,
2754                 0xe000, 0xea7c,
2755                 0xf000, 0x11190,
2756                 0x19040, 0x1906c,
2757                 0x19078, 0x19080,
2758                 0x1908c, 0x19124,
2759                 0x19150, 0x191b0,
2760                 0x191d0, 0x191e8,
2761                 0x19238, 0x1924c,
2762                 0x193f8, 0x19474,
2763                 0x19490, 0x194f8,
2764                 0x19800, 0x19f30,
2765                 0x1a000, 0x1a06c,
2766                 0x1a0b0, 0x1a120,
2767                 0x1a128, 0x1a138,
2768                 0x1a190, 0x1a1c4,
2769                 0x1a1fc, 0x1a1fc,
2770                 0x1e040, 0x1e04c,
2771                 0x1e284, 0x1e28c,
2772                 0x1e2c0, 0x1e2c0,
2773                 0x1e2e0, 0x1e2e0,
2774                 0x1e300, 0x1e384,
2775                 0x1e3c0, 0x1e3c8,
2776                 0x1e440, 0x1e44c,
2777                 0x1e684, 0x1e68c,
2778                 0x1e6c0, 0x1e6c0,
2779                 0x1e6e0, 0x1e6e0,
2780                 0x1e700, 0x1e784,
2781                 0x1e7c0, 0x1e7c8,
2782                 0x1e840, 0x1e84c,
2783                 0x1ea84, 0x1ea8c,
2784                 0x1eac0, 0x1eac0,
2785                 0x1eae0, 0x1eae0,
2786                 0x1eb00, 0x1eb84,
2787                 0x1ebc0, 0x1ebc8,
2788                 0x1ec40, 0x1ec4c,
2789                 0x1ee84, 0x1ee8c,
2790                 0x1eec0, 0x1eec0,
2791                 0x1eee0, 0x1eee0,
2792                 0x1ef00, 0x1ef84,
2793                 0x1efc0, 0x1efc8,
2794                 0x1f040, 0x1f04c,
2795                 0x1f284, 0x1f28c,
2796                 0x1f2c0, 0x1f2c0,
2797                 0x1f2e0, 0x1f2e0,
2798                 0x1f300, 0x1f384,
2799                 0x1f3c0, 0x1f3c8,
2800                 0x1f440, 0x1f44c,
2801                 0x1f684, 0x1f68c,
2802                 0x1f6c0, 0x1f6c0,
2803                 0x1f6e0, 0x1f6e0,
2804                 0x1f700, 0x1f784,
2805                 0x1f7c0, 0x1f7c8,
2806                 0x1f840, 0x1f84c,
2807                 0x1fa84, 0x1fa8c,
2808                 0x1fac0, 0x1fac0,
2809                 0x1fae0, 0x1fae0,
2810                 0x1fb00, 0x1fb84,
2811                 0x1fbc0, 0x1fbc8,
2812                 0x1fc40, 0x1fc4c,
2813                 0x1fe84, 0x1fe8c,
2814                 0x1fec0, 0x1fec0,
2815                 0x1fee0, 0x1fee0,
2816                 0x1ff00, 0x1ff84,
2817                 0x1ffc0, 0x1ffc8,
2818                 0x20000, 0x2002c,
2819                 0x20100, 0x2013c,
2820                 0x20190, 0x201c8,
2821                 0x20200, 0x20318,
2822                 0x20400, 0x20528,
2823                 0x20540, 0x20614,
2824                 0x21000, 0x21040,
2825                 0x2104c, 0x21060,
2826                 0x210c0, 0x210ec,
2827                 0x21200, 0x21268,
2828                 0x21270, 0x21284,
2829                 0x212fc, 0x21388,
2830                 0x21400, 0x21404,
2831                 0x21500, 0x21518,
2832                 0x2152c, 0x2153c,
2833                 0x21550, 0x21554,
2834                 0x21600, 0x21600,
2835                 0x21608, 0x21628,
2836                 0x21630, 0x2163c,
2837                 0x21700, 0x2171c,
2838                 0x21780, 0x2178c,
2839                 0x21800, 0x21c38,
2840                 0x21c80, 0x21d7c,
2841                 0x21e00, 0x21e04,
2842                 0x22000, 0x2202c,
2843                 0x22100, 0x2213c,
2844                 0x22190, 0x221c8,
2845                 0x22200, 0x22318,
2846                 0x22400, 0x22528,
2847                 0x22540, 0x22614,
2848                 0x23000, 0x23040,
2849                 0x2304c, 0x23060,
2850                 0x230c0, 0x230ec,
2851                 0x23200, 0x23268,
2852                 0x23270, 0x23284,
2853                 0x232fc, 0x23388,
2854                 0x23400, 0x23404,
2855                 0x23500, 0x23518,
2856                 0x2352c, 0x2353c,
2857                 0x23550, 0x23554,
2858                 0x23600, 0x23600,
2859                 0x23608, 0x23628,
2860                 0x23630, 0x2363c,
2861                 0x23700, 0x2371c,
2862                 0x23780, 0x2378c,
2863                 0x23800, 0x23c38,
2864                 0x23c80, 0x23d7c,
2865                 0x23e00, 0x23e04,
2866                 0x24000, 0x2402c,
2867                 0x24100, 0x2413c,
2868                 0x24190, 0x241c8,
2869                 0x24200, 0x24318,
2870                 0x24400, 0x24528,
2871                 0x24540, 0x24614,
2872                 0x25000, 0x25040,
2873                 0x2504c, 0x25060,
2874                 0x250c0, 0x250ec,
2875                 0x25200, 0x25268,
2876                 0x25270, 0x25284,
2877                 0x252fc, 0x25388,
2878                 0x25400, 0x25404,
2879                 0x25500, 0x25518,
2880                 0x2552c, 0x2553c,
2881                 0x25550, 0x25554,
2882                 0x25600, 0x25600,
2883                 0x25608, 0x25628,
2884                 0x25630, 0x2563c,
2885                 0x25700, 0x2571c,
2886                 0x25780, 0x2578c,
2887                 0x25800, 0x25c38,
2888                 0x25c80, 0x25d7c,
2889                 0x25e00, 0x25e04,
2890                 0x26000, 0x2602c,
2891                 0x26100, 0x2613c,
2892                 0x26190, 0x261c8,
2893                 0x26200, 0x26318,
2894                 0x26400, 0x26528,
2895                 0x26540, 0x26614,
2896                 0x27000, 0x27040,
2897                 0x2704c, 0x27060,
2898                 0x270c0, 0x270ec,
2899                 0x27200, 0x27268,
2900                 0x27270, 0x27284,
2901                 0x272fc, 0x27388,
2902                 0x27400, 0x27404,
2903                 0x27500, 0x27518,
2904                 0x2752c, 0x2753c,
2905                 0x27550, 0x27554,
2906                 0x27600, 0x27600,
2907                 0x27608, 0x27628,
2908                 0x27630, 0x2763c,
2909                 0x27700, 0x2771c,
2910                 0x27780, 0x2778c,
2911                 0x27800, 0x27c38,
2912                 0x27c80, 0x27d7c,
2913                 0x27e00, 0x27e04
2914         };
2915
2916         regs->version = 4 | (sc->params.rev << 10);
2917         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
2918                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
2919 }
2920
2921 static void
2922 cxgbe_tick(void *arg)
2923 {
2924         struct port_info *pi = arg;
2925         struct ifnet *ifp = pi->ifp;
2926         struct sge_txq *txq;
2927         int i, drops;
2928         struct port_stats *s = &pi->stats;
2929
2930         PORT_LOCK(pi);
2931         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2932                 PORT_UNLOCK(pi);
2933                 return; /* without scheduling another callout */
2934         }
2935
2936         t4_get_port_stats(pi->adapter, pi->tx_chan, s);
2937
2938         ifp->if_opackets = s->tx_frames - s->tx_pause;
2939         ifp->if_ipackets = s->rx_frames - s->rx_pause;
2940         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
2941         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
2942         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
2943         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
2944         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2945             s->rx_ovflow3;
2946
2947         drops = s->tx_drop;
2948         for_each_txq(pi, i, txq)
2949                 drops += txq->br->br_drops;
2950         ifp->if_snd.ifq_drops = drops;
2951
2952         ifp->if_oerrors = s->tx_error_frames;
2953         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
2954             s->rx_fcs_err + s->rx_len_err;
2955
2956         callout_schedule(&pi->tick, hz);
2957         PORT_UNLOCK(pi);
2958 }
2959
2960 static void
2961 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
2962 {
2963         struct ifnet *vlan;
2964
2965         if (arg != ifp)
2966                 return;
2967
2968         vlan = VLAN_DEVAT(ifp, vid);
2969         VLAN_SETCOOKIE(vlan, ifp);
2970 }
2971
2972 static int
2973 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
2974 {
2975
2976 #ifdef INVARIANTS
2977         panic("%s: opcode 0x%02x on iq %p with payload %p",
2978             __func__, rss->opcode, iq, m);
2979 #else
2980         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p",
2981             __func__, rss->opcode, iq, m);
2982         m_freem(m);
2983 #endif
2984         return (EDOOFUS);
2985 }
2986
2987 int
2988 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2989 {
2990         uintptr_t *loc, new;
2991
2992         if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2993                 return (EINVAL);
2994
2995         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
2996         loc = (uintptr_t *) &sc->cpl_handler[opcode];
2997         atomic_store_rel_ptr(loc, new);
2998
2999         return (0);
3000 }
3001
3002 static int
3003 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
3004 {
3005
3006 #ifdef INVARIANTS
3007         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
3008 #else
3009         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)",
3010             __func__, iq, ctrl);
3011 #endif
3012         return (EDOOFUS);
3013 }
3014
3015 int
3016 t4_register_an_handler(struct adapter *sc, an_handler_t h)
3017 {
3018         uintptr_t *loc, new;
3019
3020         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
3021         loc = (uintptr_t *) &sc->an_handler;
3022         atomic_store_rel_ptr(loc, new);
3023
3024         return (0);
3025 }
3026
3027 static int
3028 t4_sysctls(struct adapter *sc)
3029 {
3030         struct sysctl_ctx_list *ctx;
3031         struct sysctl_oid *oid;
3032         struct sysctl_oid_list *children, *c0;
3033         static char *caps[] = {
3034                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
3035                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",       /* caps[1] niccaps */
3036                 "\20\1TOE",                             /* caps[2] toecaps */
3037                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
3038                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
3039                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
3040                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
3041                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
3042         };
3043
3044         ctx = device_get_sysctl_ctx(sc->dev);
3045
3046         /*
3047          * dev.t4nex.X.
3048          */
3049         oid = device_get_sysctl_tree(sc->dev);
3050         c0 = children = SYSCTL_CHILDREN(oid);
3051
3052         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3053             &sc->params.nports, 0, "# of ports");
3054
3055         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3056             &sc->params.rev, 0, "chip hardware revision");
3057
3058         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3059             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3060
3061         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3062             CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
3063
3064         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3065             &sc->cfcsum, 0, "config file checksum");
3066
3067         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3068             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3069             sysctl_bitfield, "A", "available link capabilities");
3070
3071         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3072             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3073             sysctl_bitfield, "A", "available NIC capabilities");
3074
3075         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3076             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3077             sysctl_bitfield, "A", "available TCP offload capabilities");
3078
3079         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3080             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3081             sysctl_bitfield, "A", "available RDMA capabilities");
3082
3083         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3084             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3085             sysctl_bitfield, "A", "available iSCSI capabilities");
3086
3087         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3088             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3089             sysctl_bitfield, "A", "available FCoE capabilities");
3090
3091         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3092             &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3093
3094         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3095             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3096             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3097             "interrupt holdoff timer values (us)");
3098
3099         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3100             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3101             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3102             "interrupt holdoff packet counter values");
3103
3104 #ifdef SBUF_DRAIN
3105         /*
3106          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3107          */
3108         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3109             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3110             "logs and miscellaneous information");
3111         children = SYSCTL_CHILDREN(oid);
3112
3113         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3114             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3115             sysctl_cctrl, "A", "congestion control");
3116
3117         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3118             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3119             sysctl_cpl_stats, "A", "CPL statistics");
3120
3121         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3122             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3123             sysctl_ddp_stats, "A", "DDP statistics");
3124
3125         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3126             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3127             sysctl_devlog, "A", "firmware's device log");
3128
3129         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3130             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3131             sysctl_fcoe_stats, "A", "FCoE statistics");
3132
3133         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3134             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3135             sysctl_hw_sched, "A", "hardware scheduler ");
3136
3137         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3138             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3139             sysctl_l2t, "A", "hardware L2 table");
3140
3141         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3142             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3143             sysctl_lb_stats, "A", "loopback statistics");
3144
3145         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3146             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3147             sysctl_meminfo, "A", "memory regions");
3148
3149         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3150             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3151             sysctl_path_mtus, "A", "path MTUs");
3152
3153         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3154             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3155             sysctl_pm_stats, "A", "PM statistics");
3156
3157         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3158             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3159             sysctl_rdma_stats, "A", "RDMA statistics");
3160
3161         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3162             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3163             sysctl_tcp_stats, "A", "TCP statistics");
3164
3165         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3166             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3167             sysctl_tids, "A", "TID information");
3168
3169         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3170             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3171             sysctl_tp_err_stats, "A", "TP error statistics");
3172
3173         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3174             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3175             sysctl_tx_rate, "A", "Tx rate");
3176 #endif
3177
3178 #ifdef TCP_OFFLOAD
3179         if (is_offload(sc)) {
3180                 /*
3181                  * dev.t4nex.X.toe.
3182                  */
3183                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3184                     NULL, "TOE parameters");
3185                 children = SYSCTL_CHILDREN(oid);
3186
3187                 sc->tt.sndbuf = 256 * 1024;
3188                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3189                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
3190
3191                 sc->tt.ddp = 0;
3192                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3193                     &sc->tt.ddp, 0, "DDP allowed");
3194                 sc->tt.indsz = M_INDICATESIZE;
3195                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3196                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
3197                 sc->tt.ddp_thres = 3*4096;
3198                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3199                     &sc->tt.ddp_thres, 0, "DDP threshold");
3200         }
3201 #endif
3202
3203
3204         return (0);
3205 }
3206
3207 static int
3208 cxgbe_sysctls(struct port_info *pi)
3209 {
3210         struct sysctl_ctx_list *ctx;
3211         struct sysctl_oid *oid;
3212         struct sysctl_oid_list *children;
3213
3214         ctx = device_get_sysctl_ctx(pi->dev);
3215
3216         /*
3217          * dev.cxgbe.X.
3218          */
3219         oid = device_get_sysctl_tree(pi->dev);
3220         children = SYSCTL_CHILDREN(oid);
3221
3222         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3223             &pi->nrxq, 0, "# of rx queues");
3224         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3225             &pi->ntxq, 0, "# of tx queues");
3226         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3227             &pi->first_rxq, 0, "index of first rx queue");
3228         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3229             &pi->first_txq, 0, "index of first tx queue");
3230
3231 #ifdef TCP_OFFLOAD
3232         if (is_offload(pi->adapter)) {
3233                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3234                     &pi->nofldrxq, 0,
3235                     "# of rx queues for offloaded TCP connections");
3236                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3237                     &pi->nofldtxq, 0,
3238                     "# of tx queues for offloaded TCP connections");
3239                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3240                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3241                     "index of first TOE rx queue");
3242                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3243                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
3244                     "index of first TOE tx queue");
3245         }
3246 #endif
3247
3248         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3249             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3250             "holdoff timer index");
3251         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3252             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3253             "holdoff packet counter index");
3254
3255         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3256             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3257             "rx queue size");
3258         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3259             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3260             "tx queue size");
3261
3262         /*
3263          * dev.cxgbe.X.stats.
3264          */
3265         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3266             NULL, "port statistics");
3267         children = SYSCTL_CHILDREN(oid);
3268
3269 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3270         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3271             CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3272             sysctl_handle_t4_reg64, "QU", desc)
3273
3274         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3275             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3276         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3277             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3278         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3279             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3280         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3281             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3282         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3283             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3284         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3285             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3286         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3287             "# of tx frames in this range",
3288             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3289         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3290             "# of tx frames in this range",
3291             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3292         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3293             "# of tx frames in this range",
3294             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3295         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3296             "# of tx frames in this range",
3297             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3298         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3299             "# of tx frames in this range",
3300             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3301         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3302             "# of tx frames in this range",
3303             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3304         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3305             "# of tx frames in this range",
3306             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3307         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3308             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3309         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3310             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3311         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3312             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3313         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3314             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3315         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3316             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3317         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3318             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3319         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3320             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3321         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3322             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3323         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3324             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3325         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3326             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3327
3328         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3329             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3330         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3331             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3332         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3333             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3334         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3335             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3336         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3337             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3338         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3339             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3340         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3341             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3342         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3343             "# of frames received with bad FCS",
3344             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3345         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3346             "# of frames received with length error",
3347             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3348         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3349             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3350         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3351             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3352         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3353             "# of rx frames in this range",
3354             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3355         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3356             "# of rx frames in this range",
3357             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3358         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3359             "# of rx frames in this range",
3360             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3361         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3362             "# of rx frames in this range",
3363             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3364         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3365             "# of rx frames in this range",
3366             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3367         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3368             "# of rx frames in this range",
3369             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3370         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3371             "# of rx frames in this range",
3372             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3373         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3374             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3375         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3376             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3377         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3378             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3379         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3380             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3381         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3382             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3383         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3384             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3385         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3386             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3387         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3388             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3389         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3390             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3391
3392 #undef SYSCTL_ADD_T4_REG64
3393
3394 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3395         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3396             &pi->stats.name, desc)
3397
3398         /* We get these from port_stats and they may be stale by upto 1s */
3399         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3400             "# drops due to buffer-group 0 overflows");
3401         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3402             "# drops due to buffer-group 1 overflows");
3403         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3404             "# drops due to buffer-group 2 overflows");
3405         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3406             "# drops due to buffer-group 3 overflows");
3407         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3408             "# of buffer-group 0 truncated packets");
3409         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3410             "# of buffer-group 1 truncated packets");
3411         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3412             "# of buffer-group 2 truncated packets");
3413         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3414             "# of buffer-group 3 truncated packets");
3415
3416 #undef SYSCTL_ADD_T4_PORTSTAT
3417
3418         return (0);
3419 }
3420
3421 static int
3422 sysctl_int_array(SYSCTL_HANDLER_ARGS)
3423 {
3424         int rc, *i;
3425         struct sbuf sb;
3426
3427         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3428         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3429                 sbuf_printf(&sb, "%d ", *i);
3430         sbuf_trim(&sb);
3431         sbuf_finish(&sb);
3432         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3433         sbuf_delete(&sb);
3434         return (rc);
3435 }
3436
3437 static int
3438 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
3439 {
3440         int rc;
3441         struct sbuf *sb;
3442
3443         rc = sysctl_wire_old_buffer(req, 0);
3444         if (rc != 0)
3445                 return(rc);
3446
3447         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3448         if (sb == NULL)
3449                 return (ENOMEM);
3450
3451         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3452         rc = sbuf_finish(sb);
3453         sbuf_delete(sb);
3454
3455         return (rc);
3456 }
3457
3458 static int
3459 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3460 {
3461         struct port_info *pi = arg1;
3462         struct adapter *sc = pi->adapter;
3463         int idx, rc, i;
3464
3465         idx = pi->tmr_idx;
3466
3467         rc = sysctl_handle_int(oidp, &idx, 0, req);
3468         if (rc != 0 || req->newptr == NULL)
3469                 return (rc);
3470
3471         if (idx < 0 || idx >= SGE_NTIMERS)
3472                 return (EINVAL);
3473
3474         ADAPTER_LOCK(sc);
3475         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3476         if (rc == 0) {
3477                 struct sge_rxq *rxq;
3478                 uint8_t v;
3479
3480                 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3481                 for_each_rxq(pi, i, rxq) {
3482 #ifdef atomic_store_rel_8
3483                         atomic_store_rel_8(&rxq->iq.intr_params, v);
3484 #else
3485                         rxq->iq.intr_params = v;
3486 #endif
3487                 }
3488                 pi->tmr_idx = idx;
3489         }
3490
3491         ADAPTER_UNLOCK(sc);
3492         return (rc);
3493 }
3494
3495 static int
3496 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3497 {
3498         struct port_info *pi = arg1;
3499         struct adapter *sc = pi->adapter;
3500         int idx, rc;
3501
3502         idx = pi->pktc_idx;
3503
3504         rc = sysctl_handle_int(oidp, &idx, 0, req);
3505         if (rc != 0 || req->newptr == NULL)
3506                 return (rc);
3507
3508         if (idx < -1 || idx >= SGE_NCOUNTERS)
3509                 return (EINVAL);
3510
3511         ADAPTER_LOCK(sc);
3512         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3513         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3514                 rc = EBUSY; /* cannot be changed once the queues are created */
3515
3516         if (rc == 0)
3517                 pi->pktc_idx = idx;
3518
3519         ADAPTER_UNLOCK(sc);
3520         return (rc);
3521 }
3522
3523 static int
3524 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3525 {
3526         struct port_info *pi = arg1;
3527         struct adapter *sc = pi->adapter;
3528         int qsize, rc;
3529
3530         qsize = pi->qsize_rxq;
3531
3532         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3533         if (rc != 0 || req->newptr == NULL)
3534                 return (rc);
3535
3536         if (qsize < 128 || (qsize & 7))
3537                 return (EINVAL);
3538
3539         ADAPTER_LOCK(sc);
3540         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3541         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3542                 rc = EBUSY; /* cannot be changed once the queues are created */
3543
3544         if (rc == 0)
3545                 pi->qsize_rxq = qsize;
3546
3547         ADAPTER_UNLOCK(sc);
3548         return (rc);
3549 }
3550
3551 static int
3552 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3553 {
3554         struct port_info *pi = arg1;
3555         struct adapter *sc = pi->adapter;
3556         int qsize, rc;
3557
3558         qsize = pi->qsize_txq;
3559
3560         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3561         if (rc != 0 || req->newptr == NULL)
3562                 return (rc);
3563
3564         if (qsize < 128)
3565                 return (EINVAL);
3566
3567         ADAPTER_LOCK(sc);
3568         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3569         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3570                 rc = EBUSY; /* cannot be changed once the queues are created */
3571
3572         if (rc == 0)
3573                 pi->qsize_txq = qsize;
3574
3575         ADAPTER_UNLOCK(sc);
3576         return (rc);
3577 }
3578
3579 static int
3580 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3581 {
3582         struct adapter *sc = arg1;
3583         int reg = arg2;
3584         uint64_t val;
3585
3586         val = t4_read_reg64(sc, reg);
3587
3588         return (sysctl_handle_64(oidp, &val, 0, req));
3589 }
3590
3591 #ifdef SBUF_DRAIN
3592 static int
3593 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
3594 {
3595         struct adapter *sc = arg1;
3596         struct sbuf *sb;
3597         int rc, i;
3598         uint16_t incr[NMTUS][NCCTRL_WIN];
3599         static const char *dec_fac[] = {
3600                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3601                 "0.9375"
3602         };
3603
3604         rc = sysctl_wire_old_buffer(req, 0);
3605         if (rc != 0)
3606                 return (rc);
3607
3608         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3609         if (sb == NULL)
3610                 return (ENOMEM);
3611
3612         t4_read_cong_tbl(sc, incr);
3613
3614         for (i = 0; i < NCCTRL_WIN; ++i) {
3615                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3616                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3617                     incr[5][i], incr[6][i], incr[7][i]);
3618                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3619                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3620                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3621                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3622         }
3623
3624         rc = sbuf_finish(sb);
3625         sbuf_delete(sb);
3626
3627         return (rc);
3628 }
3629
3630 static int
3631 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
3632 {
3633         struct adapter *sc = arg1;
3634         struct sbuf *sb;
3635         int rc;
3636         struct tp_cpl_stats stats;
3637
3638         rc = sysctl_wire_old_buffer(req, 0);
3639         if (rc != 0)
3640                 return (rc);
3641
3642         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3643         if (sb == NULL)
3644                 return (ENOMEM);
3645
3646         t4_tp_get_cpl_stats(sc, &stats);
3647
3648         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
3649             "channel 3\n");
3650         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
3651                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
3652         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
3653                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
3654
3655         rc = sbuf_finish(sb);
3656         sbuf_delete(sb);
3657
3658         return (rc);
3659 }
3660
3661 static int
3662 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
3663 {
3664         struct adapter *sc = arg1;
3665         struct sbuf *sb;
3666         int rc;
3667         struct tp_usm_stats stats;
3668
3669         rc = sysctl_wire_old_buffer(req, 0);
3670         if (rc != 0)
3671                 return(rc);
3672
3673         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3674         if (sb == NULL)
3675                 return (ENOMEM);
3676
3677         t4_get_usm_stats(sc, &stats);
3678
3679         sbuf_printf(sb, "Frames: %u\n", stats.frames);
3680         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
3681         sbuf_printf(sb, "Drops:  %u", stats.drops);
3682
3683         rc = sbuf_finish(sb);
3684         sbuf_delete(sb);
3685
3686         return (rc);
3687 }
3688
3689 const char *devlog_level_strings[] = {
3690         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
3691         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
3692         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
3693         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
3694         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
3695         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
3696 };
3697
3698 const char *devlog_facility_strings[] = {
3699         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
3700         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
3701         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
3702         [FW_DEVLOG_FACILITY_RES]        = "RES",
3703         [FW_DEVLOG_FACILITY_HW]         = "HW",
3704         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
3705         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
3706         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
3707         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
3708         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
3709         [FW_DEVLOG_FACILITY_VI]         = "VI",
3710         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
3711         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
3712         [FW_DEVLOG_FACILITY_TM]         = "TM",
3713         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
3714         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
3715         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
3716         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
3717         [FW_DEVLOG_FACILITY_RI]         = "RI",
3718         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
3719         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
3720         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
3721         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
3722 };
3723
3724 static int
3725 sysctl_devlog(SYSCTL_HANDLER_ARGS)
3726 {
3727         struct adapter *sc = arg1;
3728         struct devlog_params *dparams = &sc->params.devlog;
3729         struct fw_devlog_e *buf, *e;
3730         int i, j, rc, nentries, first = 0;
3731         struct sbuf *sb;
3732         uint64_t ftstamp = UINT64_MAX;
3733
3734         if (dparams->start == 0)
3735                 return (ENXIO);
3736
3737         nentries = dparams->size / sizeof(struct fw_devlog_e);
3738
3739         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
3740         if (buf == NULL)
3741                 return (ENOMEM);
3742
3743         rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
3744             (void *)buf);
3745         if (rc != 0)
3746                 goto done;
3747
3748         for (i = 0; i < nentries; i++) {
3749                 e = &buf[i];
3750
3751                 if (e->timestamp == 0)
3752                         break;  /* end */
3753
3754                 e->timestamp = be64toh(e->timestamp);
3755                 e->seqno = be32toh(e->seqno);
3756                 for (j = 0; j < 8; j++)
3757                         e->params[j] = be32toh(e->params[j]);
3758
3759                 if (e->timestamp < ftstamp) {
3760                         ftstamp = e->timestamp;
3761                         first = i;
3762                 }
3763         }
3764
3765         if (buf[first].timestamp == 0)
3766                 goto done;      /* nothing in the log */
3767
3768         rc = sysctl_wire_old_buffer(req, 0);
3769         if (rc != 0)
3770                 goto done;
3771
3772         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3773         if (sb == NULL) {
3774                 rc = ENOMEM;
3775                 goto done;
3776         }
3777         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
3778             "Seq#", "Tstamp", "Level", "Facility", "Message");
3779
3780         i = first;
3781         do {
3782                 e = &buf[i];
3783                 if (e->timestamp == 0)
3784                         break;  /* end */
3785
3786                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
3787                     e->seqno, e->timestamp,
3788                     (e->level < ARRAY_SIZE(devlog_level_strings) ?
3789                         devlog_level_strings[e->level] : "UNKNOWN"),
3790                     (e->facility < ARRAY_SIZE(devlog_facility_strings) ?
3791                         devlog_facility_strings[e->facility] : "UNKNOWN"));
3792                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
3793                     e->params[2], e->params[3], e->params[4],
3794                     e->params[5], e->params[6], e->params[7]);
3795
3796                 if (++i == nentries)
3797                         i = 0;
3798         } while (i != first);
3799
3800         rc = sbuf_finish(sb);
3801         sbuf_delete(sb);
3802 done:
3803         free(buf, M_CXGBE);
3804         return (rc);
3805 }
3806
3807 static int
3808 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
3809 {
3810         struct adapter *sc = arg1;
3811         struct sbuf *sb;
3812         int rc;
3813         struct tp_fcoe_stats stats[4];
3814
3815         rc = sysctl_wire_old_buffer(req, 0);
3816         if (rc != 0)
3817                 return (rc);
3818
3819         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3820         if (sb == NULL)
3821                 return (ENOMEM);
3822
3823         t4_get_fcoe_stats(sc, 0, &stats[0]);
3824         t4_get_fcoe_stats(sc, 1, &stats[1]);
3825         t4_get_fcoe_stats(sc, 2, &stats[2]);
3826         t4_get_fcoe_stats(sc, 3, &stats[3]);
3827
3828         sbuf_printf(sb, "                   channel 0        channel 1        "
3829             "channel 2        channel 3\n");
3830         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
3831             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
3832             stats[3].octetsDDP);
3833         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
3834             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
3835         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
3836             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
3837             stats[3].framesDrop);
3838
3839         rc = sbuf_finish(sb);
3840         sbuf_delete(sb);
3841
3842         return (rc);
3843 }
3844
3845 static int
3846 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
3847 {
3848         struct adapter *sc = arg1;
3849         struct sbuf *sb;
3850         int rc, i;
3851         unsigned int map, kbps, ipg, mode;
3852         unsigned int pace_tab[NTX_SCHED];
3853
3854         rc = sysctl_wire_old_buffer(req, 0);
3855         if (rc != 0)
3856                 return (rc);
3857
3858         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3859         if (sb == NULL)
3860                 return (ENOMEM);
3861
3862         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
3863         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
3864         t4_read_pace_tbl(sc, pace_tab);
3865
3866         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
3867             "Class IPG (0.1 ns)   Flow IPG (us)");
3868
3869         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
3870                 t4_get_tx_sched(sc, i, &kbps, &ipg);
3871                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
3872                     (mode & (1 << i)) ? "flow" : "class", map & 3);
3873                 if (kbps)
3874                         sbuf_printf(sb, "%9u     ", kbps);
3875                 else
3876                         sbuf_printf(sb, " disabled     ");
3877
3878                 if (ipg)
3879                         sbuf_printf(sb, "%13u        ", ipg);
3880                 else
3881                         sbuf_printf(sb, "     disabled        ");
3882
3883                 if (pace_tab[i])
3884                         sbuf_printf(sb, "%10u", pace_tab[i]);
3885                 else
3886                         sbuf_printf(sb, "  disabled");
3887         }
3888
3889         rc = sbuf_finish(sb);
3890         sbuf_delete(sb);
3891
3892         return (rc);
3893 }
3894
3895 static int
3896 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
3897 {
3898         struct adapter *sc = arg1;
3899         struct sbuf *sb;
3900         int rc, i, j;
3901         uint64_t *p0, *p1;
3902         struct lb_port_stats s[2];
3903         static const char *stat_name[] = {
3904                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
3905                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
3906                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
3907                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
3908                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
3909                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
3910                 "BG2FramesTrunc:", "BG3FramesTrunc:"
3911         };
3912
3913         rc = sysctl_wire_old_buffer(req, 0);
3914         if (rc != 0)
3915                 return (rc);
3916
3917         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3918         if (sb == NULL)
3919                 return (ENOMEM);
3920
3921         memset(s, 0, sizeof(s));
3922
3923         for (i = 0; i < 4; i += 2) {
3924                 t4_get_lb_stats(sc, i, &s[0]);
3925                 t4_get_lb_stats(sc, i + 1, &s[1]);
3926
3927                 p0 = &s[0].octets;
3928                 p1 = &s[1].octets;
3929                 sbuf_printf(sb, "%s                       Loopback %u"
3930                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
3931
3932                 for (j = 0; j < ARRAY_SIZE(stat_name); j++)
3933                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
3934                                    *p0++, *p1++);
3935         }
3936
3937         rc = sbuf_finish(sb);
3938         sbuf_delete(sb);
3939
3940         return (rc);
3941 }
3942
3943 struct mem_desc {
3944         unsigned int base;
3945         unsigned int limit;
3946         unsigned int idx;
3947 };
3948
3949 static int
3950 mem_desc_cmp(const void *a, const void *b)
3951 {
3952         return ((const struct mem_desc *)a)->base -
3953                ((const struct mem_desc *)b)->base;
3954 }
3955
3956 static void
3957 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
3958     unsigned int to)
3959 {
3960         unsigned int size;
3961
3962         size = to - from + 1;
3963         if (size == 0)
3964                 return;
3965
3966         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
3967         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
3968 }
3969
3970 static int
3971 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
3972 {
3973         struct adapter *sc = arg1;
3974         struct sbuf *sb;
3975         int rc, i, n;
3976         uint32_t lo, hi;
3977         static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
3978         static const char *region[] = {
3979                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
3980                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
3981                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
3982                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
3983                 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
3984                 "ULPTX state:", "On-chip queues:"
3985         };
3986         struct mem_desc avail[3];
3987         struct mem_desc mem[ARRAY_SIZE(region) + 3];    /* up to 3 holes */
3988         struct mem_desc *md = mem;
3989
3990         rc = sysctl_wire_old_buffer(req, 0);
3991         if (rc != 0)
3992                 return (rc);
3993
3994         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3995         if (sb == NULL)
3996                 return (ENOMEM);
3997
3998         for (i = 0; i < ARRAY_SIZE(mem); i++) {
3999                 mem[i].limit = 0;
4000                 mem[i].idx = i;
4001         }
4002
4003         /* Find and sort the populated memory ranges */
4004         i = 0;
4005         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4006         if (lo & F_EDRAM0_ENABLE) {
4007                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4008                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
4009                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
4010                 avail[i].idx = 0;
4011                 i++;
4012         }
4013         if (lo & F_EDRAM1_ENABLE) {
4014                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4015                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
4016                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
4017                 avail[i].idx = 1;
4018                 i++;
4019         }
4020         if (lo & F_EXT_MEM_ENABLE) {
4021                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4022                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
4023                 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
4024                 avail[i].idx = 2;
4025                 i++;
4026         }
4027         if (!i)                                    /* no memory available */
4028                 return 0;
4029         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
4030
4031         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
4032         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
4033         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
4034         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4035         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
4036         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
4037         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
4038         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
4039         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
4040
4041         /* the next few have explicit upper bounds */
4042         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
4043         md->limit = md->base - 1 +
4044                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
4045                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
4046         md++;
4047
4048         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4049         md->limit = md->base - 1 +
4050                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4051                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4052         md++;
4053
4054         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4055                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4056                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4057                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4058         } else {
4059                 md->base = 0;
4060                 md->idx = ARRAY_SIZE(region);  /* hide it */
4061         }
4062         md++;
4063
4064 #define ulp_region(reg) \
4065         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4066         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4067
4068         ulp_region(RX_ISCSI);
4069         ulp_region(RX_TDDP);
4070         ulp_region(TX_TPT);
4071         ulp_region(RX_STAG);
4072         ulp_region(RX_RQ);
4073         ulp_region(RX_RQUDP);
4074         ulp_region(RX_PBL);
4075         ulp_region(TX_PBL);
4076 #undef ulp_region
4077
4078         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4079         md->limit = md->base + sc->tids.ntids - 1;
4080         md++;
4081         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4082         md->limit = md->base + sc->tids.ntids - 1;
4083         md++;
4084
4085         md->base = sc->vres.ocq.start;
4086         if (sc->vres.ocq.size)
4087                 md->limit = md->base + sc->vres.ocq.size - 1;
4088         else
4089                 md->idx = ARRAY_SIZE(region);  /* hide it */
4090         md++;
4091
4092         /* add any address-space holes, there can be up to 3 */
4093         for (n = 0; n < i - 1; n++)
4094                 if (avail[n].limit < avail[n + 1].base)
4095                         (md++)->base = avail[n].limit;
4096         if (avail[n].limit)
4097                 (md++)->base = avail[n].limit;
4098
4099         n = md - mem;
4100         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4101
4102         for (lo = 0; lo < i; lo++)
4103                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4104                                 avail[lo].limit - 1);
4105
4106         sbuf_printf(sb, "\n");
4107         for (i = 0; i < n; i++) {
4108                 if (mem[i].idx >= ARRAY_SIZE(region))
4109                         continue;                        /* skip holes */
4110                 if (!mem[i].limit)
4111                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4112                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
4113                                 mem[i].limit);
4114         }
4115
4116         sbuf_printf(sb, "\n");
4117         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4118         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4119         mem_region_show(sb, "uP RAM:", lo, hi);
4120
4121         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4122         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4123         mem_region_show(sb, "uP Extmem2:", lo, hi);
4124
4125         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4126         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4127                    G_PMRXMAXPAGE(lo),
4128                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4129                    (lo & F_PMRXNUMCHN) ? 2 : 1);
4130
4131         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4132         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4133         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4134                    G_PMTXMAXPAGE(lo),
4135                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4136                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4137         sbuf_printf(sb, "%u p-structs\n",
4138                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4139
4140         for (i = 0; i < 4; i++) {
4141                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4142                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4143                            i, G_USED(lo), G_ALLOC(lo));
4144         }
4145         for (i = 0; i < 4; i++) {
4146                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4147                 sbuf_printf(sb,
4148                            "\nLoopback %d using %u pages out of %u allocated",
4149                            i, G_USED(lo), G_ALLOC(lo));
4150         }
4151
4152         rc = sbuf_finish(sb);
4153         sbuf_delete(sb);
4154
4155         return (rc);
4156 }
4157
4158 static int
4159 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4160 {
4161         struct adapter *sc = arg1;
4162         struct sbuf *sb;
4163         int rc;
4164         uint16_t mtus[NMTUS];
4165
4166         rc = sysctl_wire_old_buffer(req, 0);
4167         if (rc != 0)
4168                 return (rc);
4169
4170         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4171         if (sb == NULL)
4172                 return (ENOMEM);
4173
4174         t4_read_mtu_tbl(sc, mtus, NULL);
4175
4176         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4177             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4178             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4179             mtus[14], mtus[15]);
4180
4181         rc = sbuf_finish(sb);
4182         sbuf_delete(sb);
4183
4184         return (rc);
4185 }
4186
4187 static int
4188 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4189 {
4190         struct adapter *sc = arg1;
4191         struct sbuf *sb;
4192         int rc, i;
4193         uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4194         uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4195         static const char *pm_stats[] = {
4196                 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4197         };
4198
4199         rc = sysctl_wire_old_buffer(req, 0);
4200         if (rc != 0)
4201                 return (rc);
4202
4203         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4204         if (sb == NULL)
4205                 return (ENOMEM);
4206
4207         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4208         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4209
4210         sbuf_printf(sb, "                Tx count            Tx cycles    "
4211             "Rx count            Rx cycles");
4212         for (i = 0; i < PM_NSTATS; i++)
4213                 sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4214                     pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4215
4216         rc = sbuf_finish(sb);
4217         sbuf_delete(sb);
4218
4219         return (rc);
4220 }
4221
4222 static int
4223 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4224 {
4225         struct adapter *sc = arg1;
4226         struct sbuf *sb;
4227         int rc;
4228         struct tp_rdma_stats stats;
4229
4230         rc = sysctl_wire_old_buffer(req, 0);
4231         if (rc != 0)
4232                 return (rc);
4233
4234         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4235         if (sb == NULL)
4236                 return (ENOMEM);
4237
4238         t4_tp_get_rdma_stats(sc, &stats);
4239         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4240         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4241
4242         rc = sbuf_finish(sb);
4243         sbuf_delete(sb);
4244
4245         return (rc);
4246 }
4247
4248 static int
4249 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4250 {
4251         struct adapter *sc = arg1;
4252         struct sbuf *sb;
4253         int rc;
4254         struct tp_tcp_stats v4, v6;
4255
4256         rc = sysctl_wire_old_buffer(req, 0);
4257         if (rc != 0)
4258                 return (rc);
4259
4260         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4261         if (sb == NULL)
4262                 return (ENOMEM);
4263
4264         t4_tp_get_tcp_stats(sc, &v4, &v6);
4265         sbuf_printf(sb,
4266             "                                IP                 IPv6\n");
4267         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4268             v4.tcpOutRsts, v6.tcpOutRsts);
4269         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4270             v4.tcpInSegs, v6.tcpInSegs);
4271         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4272             v4.tcpOutSegs, v6.tcpOutSegs);
4273         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4274             v4.tcpRetransSegs, v6.tcpRetransSegs);
4275
4276         rc = sbuf_finish(sb);
4277         sbuf_delete(sb);
4278
4279         return (rc);
4280 }
4281
4282 static int
4283 sysctl_tids(SYSCTL_HANDLER_ARGS)
4284 {
4285         struct adapter *sc = arg1;
4286         struct sbuf *sb;
4287         int rc;
4288         struct tid_info *t = &sc->tids;
4289
4290         rc = sysctl_wire_old_buffer(req, 0);
4291         if (rc != 0)
4292                 return (rc);
4293
4294         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4295         if (sb == NULL)
4296                 return (ENOMEM);
4297
4298         if (t->natids) {
4299                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4300                     t->atids_in_use);
4301         }
4302
4303         if (t->ntids) {
4304                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4305                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4306
4307                         if (b) {
4308                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4309                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4310                                     t->ntids - 1);
4311                         } else {
4312                                 sbuf_printf(sb, "TID range: %u-%u",
4313                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4314                                     t->ntids - 1);
4315                         }
4316                 } else
4317                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4318                 sbuf_printf(sb, ", in use: %u\n",
4319                     atomic_load_acq_int(&t->tids_in_use));
4320         }
4321
4322         if (t->nstids) {
4323                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4324                     t->stid_base + t->nstids - 1, t->stids_in_use);
4325         }
4326
4327         if (t->nftids) {
4328                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4329                     t->ftid_base + t->nftids - 1);
4330         }
4331
4332         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4333             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4334             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4335
4336         rc = sbuf_finish(sb);
4337         sbuf_delete(sb);
4338
4339         return (rc);
4340 }
4341
4342 static int
4343 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4344 {
4345         struct adapter *sc = arg1;
4346         struct sbuf *sb;
4347         int rc;
4348         struct tp_err_stats stats;
4349
4350         rc = sysctl_wire_old_buffer(req, 0);
4351         if (rc != 0)
4352                 return (rc);
4353
4354         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4355         if (sb == NULL)
4356                 return (ENOMEM);
4357
4358         t4_tp_get_err_stats(sc, &stats);
4359
4360         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4361                       "channel 3\n");
4362         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4363             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4364             stats.macInErrs[3]);
4365         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4366             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4367             stats.hdrInErrs[3]);
4368         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4369             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4370             stats.tcpInErrs[3]);
4371         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4372             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4373             stats.tcp6InErrs[3]);
4374         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4375             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4376             stats.tnlCongDrops[3]);
4377         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4378             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4379             stats.tnlTxDrops[3]);
4380         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4381             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4382             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4383         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4384             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4385             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4386         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4387             stats.ofldNoNeigh, stats.ofldCongDefer);
4388
4389         rc = sbuf_finish(sb);
4390         sbuf_delete(sb);
4391
4392         return (rc);
4393 }
4394
4395 static int
4396 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4397 {
4398         struct adapter *sc = arg1;
4399         struct sbuf *sb;
4400         int rc;
4401         u64 nrate[NCHAN], orate[NCHAN];
4402
4403         rc = sysctl_wire_old_buffer(req, 0);
4404         if (rc != 0)
4405                 return (rc);
4406
4407         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4408         if (sb == NULL)
4409                 return (ENOMEM);
4410
4411         t4_get_chan_txrate(sc, nrate, orate);
4412         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4413                  "channel 3\n");
4414         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4415             nrate[0], nrate[1], nrate[2], nrate[3]);
4416         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4417             orate[0], orate[1], orate[2], orate[3]);
4418
4419         rc = sbuf_finish(sb);
4420         sbuf_delete(sb);
4421
4422         return (rc);
4423 }
4424 #endif
4425
4426 static inline void
4427 txq_start(struct ifnet *ifp, struct sge_txq *txq)
4428 {
4429         struct buf_ring *br;
4430         struct mbuf *m;
4431
4432         TXQ_LOCK_ASSERT_OWNED(txq);
4433
4434         br = txq->br;
4435         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4436         if (m)
4437                 t4_eth_tx(ifp, txq, m);
4438 }
4439
4440 void
4441 t4_tx_callout(void *arg)
4442 {
4443         struct sge_eq *eq = arg;
4444         struct adapter *sc;
4445
4446         if (EQ_TRYLOCK(eq) == 0)
4447                 goto reschedule;
4448
4449         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4450                 EQ_UNLOCK(eq);
4451 reschedule:
4452                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
4453                         callout_schedule(&eq->tx_callout, 1);
4454                 return;
4455         }
4456
4457         EQ_LOCK_ASSERT_OWNED(eq);
4458
4459         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4460
4461                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4462                         struct sge_txq *txq = arg;
4463                         struct port_info *pi = txq->ifp->if_softc;
4464
4465                         sc = pi->adapter;
4466                 } else {
4467                         struct sge_wrq *wrq = arg;
4468
4469                         sc = wrq->adapter;
4470                 }
4471
4472                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4473         }
4474
4475         EQ_UNLOCK(eq);
4476 }
4477
4478 void
4479 t4_tx_task(void *arg, int count)
4480 {
4481         struct sge_eq *eq = arg;
4482
4483         EQ_LOCK(eq);
4484         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4485                 struct sge_txq *txq = arg;
4486                 txq_start(txq->ifp, txq);
4487         } else {
4488                 struct sge_wrq *wrq = arg;
4489                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4490         }
4491         EQ_UNLOCK(eq);
4492 }
4493
4494 static uint32_t
4495 fconf_to_mode(uint32_t fconf)
4496 {
4497         uint32_t mode;
4498
4499         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4500             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4501
4502         if (fconf & F_FRAGMENTATION)
4503                 mode |= T4_FILTER_IP_FRAGMENT;
4504
4505         if (fconf & F_MPSHITTYPE)
4506                 mode |= T4_FILTER_MPS_HIT_TYPE;
4507
4508         if (fconf & F_MACMATCH)
4509                 mode |= T4_FILTER_MAC_IDX;
4510
4511         if (fconf & F_ETHERTYPE)
4512                 mode |= T4_FILTER_ETH_TYPE;
4513
4514         if (fconf & F_PROTOCOL)
4515                 mode |= T4_FILTER_IP_PROTO;
4516
4517         if (fconf & F_TOS)
4518                 mode |= T4_FILTER_IP_TOS;
4519
4520         if (fconf & F_VLAN)
4521                 mode |= T4_FILTER_VLAN;
4522
4523         if (fconf & F_VNIC_ID)
4524                 mode |= T4_FILTER_VNIC;
4525
4526         if (fconf & F_PORT)
4527                 mode |= T4_FILTER_PORT;
4528
4529         if (fconf & F_FCOE)
4530                 mode |= T4_FILTER_FCoE;
4531
4532         return (mode);
4533 }
4534
4535 static uint32_t
4536 mode_to_fconf(uint32_t mode)
4537 {
4538         uint32_t fconf = 0;
4539
4540         if (mode & T4_FILTER_IP_FRAGMENT)
4541                 fconf |= F_FRAGMENTATION;
4542
4543         if (mode & T4_FILTER_MPS_HIT_TYPE)
4544                 fconf |= F_MPSHITTYPE;
4545
4546         if (mode & T4_FILTER_MAC_IDX)
4547                 fconf |= F_MACMATCH;
4548
4549         if (mode & T4_FILTER_ETH_TYPE)
4550                 fconf |= F_ETHERTYPE;
4551
4552         if (mode & T4_FILTER_IP_PROTO)
4553                 fconf |= F_PROTOCOL;
4554
4555         if (mode & T4_FILTER_IP_TOS)
4556                 fconf |= F_TOS;
4557
4558         if (mode & T4_FILTER_VLAN)
4559                 fconf |= F_VLAN;
4560
4561         if (mode & T4_FILTER_VNIC)
4562                 fconf |= F_VNIC_ID;
4563
4564         if (mode & T4_FILTER_PORT)
4565                 fconf |= F_PORT;
4566
4567         if (mode & T4_FILTER_FCoE)
4568                 fconf |= F_FCOE;
4569
4570         return (fconf);
4571 }
4572
4573 static uint32_t
4574 fspec_to_fconf(struct t4_filter_specification *fs)
4575 {
4576         uint32_t fconf = 0;
4577
4578         if (fs->val.frag || fs->mask.frag)
4579                 fconf |= F_FRAGMENTATION;
4580
4581         if (fs->val.matchtype || fs->mask.matchtype)
4582                 fconf |= F_MPSHITTYPE;
4583
4584         if (fs->val.macidx || fs->mask.macidx)
4585                 fconf |= F_MACMATCH;
4586
4587         if (fs->val.ethtype || fs->mask.ethtype)
4588                 fconf |= F_ETHERTYPE;
4589
4590         if (fs->val.proto || fs->mask.proto)
4591                 fconf |= F_PROTOCOL;
4592
4593         if (fs->val.tos || fs->mask.tos)
4594                 fconf |= F_TOS;
4595
4596         if (fs->val.vlan_vld || fs->mask.vlan_vld)
4597                 fconf |= F_VLAN;
4598
4599         if (fs->val.vnic_vld || fs->mask.vnic_vld)
4600                 fconf |= F_VNIC_ID;
4601
4602         if (fs->val.iport || fs->mask.iport)
4603                 fconf |= F_PORT;
4604
4605         if (fs->val.fcoe || fs->mask.fcoe)
4606                 fconf |= F_FCOE;
4607
4608         return (fconf);
4609 }
4610
4611 static int
4612 get_filter_mode(struct adapter *sc, uint32_t *mode)
4613 {
4614         uint32_t fconf;
4615
4616         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
4617             A_TP_VLAN_PRI_MAP);
4618
4619         if (sc->filter_mode != fconf) {
4620                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
4621                     device_get_nameunit(sc->dev), sc->filter_mode, fconf);
4622                 sc->filter_mode = fconf;
4623         }
4624
4625         *mode = fconf_to_mode(sc->filter_mode);
4626
4627         return (0);
4628 }
4629
4630 static int
4631 set_filter_mode(struct adapter *sc, uint32_t mode)
4632 {
4633         uint32_t fconf;
4634         int rc;
4635
4636         fconf = mode_to_fconf(mode);
4637
4638         ADAPTER_LOCK(sc);
4639         if (IS_BUSY(sc)) {
4640                 rc = EAGAIN;
4641                 goto done;
4642         }
4643
4644         if (sc->tids.ftids_in_use > 0) {
4645                 rc = EBUSY;
4646                 goto done;
4647         }
4648
4649 #ifdef TCP_OFFLOAD
4650         if (sc->offload_map) {
4651                 rc = EBUSY;
4652                 goto done;
4653         }
4654 #endif
4655
4656 #ifdef notyet
4657         rc = -t4_set_filter_mode(sc, fconf);
4658         if (rc == 0)
4659                 sc->filter_mode = fconf;
4660 #else
4661         rc = ENOTSUP;
4662 #endif
4663
4664 done:
4665         ADAPTER_UNLOCK(sc);
4666         return (rc);
4667 }
4668
4669 static inline uint64_t
4670 get_filter_hits(struct adapter *sc, uint32_t fid)
4671 {
4672         uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4673         uint64_t hits;
4674
4675         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
4676             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
4677         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
4678         hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
4679
4680         return (be64toh(hits));
4681 }
4682
4683 static int
4684 get_filter(struct adapter *sc, struct t4_filter *t)
4685 {
4686         int i, nfilters = sc->tids.nftids;
4687         struct filter_entry *f;
4688
4689         ADAPTER_LOCK_ASSERT_OWNED(sc);
4690
4691         if (IS_BUSY(sc))
4692                 return (EAGAIN);
4693
4694         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
4695             t->idx >= nfilters) {
4696                 t->idx = 0xffffffff;
4697                 return (0);
4698         }
4699
4700         f = &sc->tids.ftid_tab[t->idx];
4701         for (i = t->idx; i < nfilters; i++, f++) {
4702                 if (f->valid) {
4703                         t->idx = i;
4704                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
4705                         t->smtidx = f->smtidx;
4706                         if (f->fs.hitcnts)
4707                                 t->hits = get_filter_hits(sc, t->idx);
4708                         else
4709                                 t->hits = UINT64_MAX;
4710                         t->fs = f->fs;
4711
4712                         return (0);
4713                 }
4714         }
4715
4716         t->idx = 0xffffffff;
4717         return (0);
4718 }
4719
4720 static int
4721 set_filter(struct adapter *sc, struct t4_filter *t)
4722 {
4723         unsigned int nfilters, nports;
4724         struct filter_entry *f;
4725         int i;
4726
4727         ADAPTER_LOCK_ASSERT_OWNED(sc);
4728
4729         nfilters = sc->tids.nftids;
4730         nports = sc->params.nports;
4731
4732         if (nfilters == 0)
4733                 return (ENOTSUP);
4734
4735         if (!(sc->flags & FULL_INIT_DONE))
4736                 return (EAGAIN);
4737
4738         if (t->idx >= nfilters)
4739                 return (EINVAL);
4740
4741         /* Validate against the global filter mode */
4742         if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode)
4743                 return (E2BIG);
4744
4745         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports)
4746                 return (EINVAL);
4747
4748         if (t->fs.val.iport >= nports)
4749                 return (EINVAL);
4750
4751         /* Can't specify an iq if not steering to it */
4752         if (!t->fs.dirsteer && t->fs.iq)
4753                 return (EINVAL);
4754
4755         /* IPv6 filter idx must be 4 aligned */
4756         if (t->fs.type == 1 &&
4757             ((t->idx & 0x3) || t->idx + 4 >= nfilters))
4758                 return (EINVAL);
4759
4760         if (sc->tids.ftid_tab == NULL) {
4761                 KASSERT(sc->tids.ftids_in_use == 0,
4762                     ("%s: no memory allocated but filters_in_use > 0",
4763                     __func__));
4764
4765                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
4766                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
4767                 if (sc->tids.ftid_tab == NULL)
4768                         return (ENOMEM);
4769         }
4770
4771         for (i = 0; i < 4; i++) {
4772                 f = &sc->tids.ftid_tab[t->idx + i];
4773
4774                 if (f->pending || f->valid)
4775                         return (EBUSY);
4776                 if (f->locked)
4777                         return (EPERM);
4778
4779                 if (t->fs.type == 0)
4780                         break;
4781         }
4782
4783         f = &sc->tids.ftid_tab[t->idx];
4784         f->fs = t->fs;
4785
4786         return set_filter_wr(sc, t->idx);
4787 }
4788
4789 static int
4790 del_filter(struct adapter *sc, struct t4_filter *t)
4791 {
4792         unsigned int nfilters;
4793         struct filter_entry *f;
4794
4795         ADAPTER_LOCK_ASSERT_OWNED(sc);
4796
4797         if (IS_BUSY(sc))
4798                 return (EAGAIN);
4799
4800         nfilters = sc->tids.nftids;
4801
4802         if (nfilters == 0)
4803                 return (ENOTSUP);
4804
4805         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
4806             t->idx >= nfilters)
4807                 return (EINVAL);
4808
4809         if (!(sc->flags & FULL_INIT_DONE))
4810                 return (EAGAIN);
4811
4812         f = &sc->tids.ftid_tab[t->idx];
4813
4814         if (f->pending)
4815                 return (EBUSY);
4816         if (f->locked)
4817                 return (EPERM);
4818
4819         if (f->valid) {
4820                 t->fs = f->fs;  /* extra info for the caller */
4821                 return del_filter_wr(sc, t->idx);
4822         }
4823
4824         return (0);
4825 }
4826
4827 static void
4828 clear_filter(struct filter_entry *f)
4829 {
4830         if (f->l2t)
4831                 t4_l2t_release(f->l2t);
4832
4833         bzero(f, sizeof (*f));
4834 }
4835
4836 static int
4837 set_filter_wr(struct adapter *sc, int fidx)
4838 {
4839         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4840         struct wrqe *wr;
4841         struct fw_filter_wr *fwr;
4842         unsigned int ftid;
4843
4844         ADAPTER_LOCK_ASSERT_OWNED(sc);
4845
4846         if (f->fs.newdmac || f->fs.newvlan) {
4847                 /* This filter needs an L2T entry; allocate one. */
4848                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
4849                 if (f->l2t == NULL)
4850                         return (EAGAIN);
4851                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
4852                     f->fs.dmac)) {
4853                         t4_l2t_release(f->l2t);
4854                         f->l2t = NULL;
4855                         return (ENOMEM);
4856                 }
4857         }
4858
4859         ftid = sc->tids.ftid_base + fidx;
4860
4861         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
4862         if (wr == NULL)
4863                 return (ENOMEM);
4864
4865         fwr = wrtod(wr);
4866         bzero(fwr, sizeof (*fwr));
4867
4868         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
4869         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
4870         fwr->tid_to_iq =
4871             htobe32(V_FW_FILTER_WR_TID(ftid) |
4872                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
4873                 V_FW_FILTER_WR_NOREPLY(0) |
4874                 V_FW_FILTER_WR_IQ(f->fs.iq));
4875         fwr->del_filter_to_l2tix =
4876             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
4877                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
4878                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
4879                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
4880                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
4881                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
4882                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
4883                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
4884                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
4885                     f->fs.newvlan == VLAN_REWRITE) |
4886                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
4887                     f->fs.newvlan == VLAN_REWRITE) |
4888                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
4889                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
4890                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
4891                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
4892         fwr->ethtype = htobe16(f->fs.val.ethtype);
4893         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
4894         fwr->frag_to_ovlan_vldm =
4895             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
4896                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
4897                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
4898                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
4899                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
4900                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
4901         fwr->smac_sel = 0;
4902         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
4903             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
4904         fwr->maci_to_matchtypem =
4905             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
4906                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
4907                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
4908                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
4909                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
4910                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
4911                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
4912                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
4913         fwr->ptcl = f->fs.val.proto;
4914         fwr->ptclm = f->fs.mask.proto;
4915         fwr->ttyp = f->fs.val.tos;
4916         fwr->ttypm = f->fs.mask.tos;
4917         fwr->ivlan = htobe16(f->fs.val.vlan);
4918         fwr->ivlanm = htobe16(f->fs.mask.vlan);
4919         fwr->ovlan = htobe16(f->fs.val.vnic);
4920         fwr->ovlanm = htobe16(f->fs.mask.vnic);
4921         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
4922         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
4923         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
4924         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
4925         fwr->lp = htobe16(f->fs.val.dport);
4926         fwr->lpm = htobe16(f->fs.mask.dport);
4927         fwr->fp = htobe16(f->fs.val.sport);
4928         fwr->fpm = htobe16(f->fs.mask.sport);
4929         if (f->fs.newsmac)
4930                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
4931
4932         f->pending = 1;
4933         sc->tids.ftids_in_use++;
4934
4935         t4_wrq_tx(sc, wr);
4936         return (0);
4937 }
4938
4939 static int
4940 del_filter_wr(struct adapter *sc, int fidx)
4941 {
4942         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4943         struct wrqe *wr;
4944         struct fw_filter_wr *fwr;
4945         unsigned int ftid;
4946
4947         ADAPTER_LOCK_ASSERT_OWNED(sc);
4948
4949         ftid = sc->tids.ftid_base + fidx;
4950
4951         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
4952         if (wr == NULL)
4953                 return (ENOMEM);
4954         fwr = wrtod(wr);
4955         bzero(fwr, sizeof (*fwr));
4956
4957         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
4958
4959         f->pending = 1;
4960         t4_wrq_tx(sc, wr);
4961         return (0);
4962 }
4963
4964 static int
4965 filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4966 {
4967         struct adapter *sc = iq->adapter;
4968         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
4969         unsigned int idx = GET_TID(rpl);
4970
4971         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
4972             rss->opcode));
4973
4974         if (idx >= sc->tids.ftid_base &&
4975             (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
4976                 unsigned int rc = G_COOKIE(rpl->cookie);
4977                 struct filter_entry *f = &sc->tids.ftid_tab[idx];
4978
4979                 ADAPTER_LOCK(sc);
4980                 if (rc == FW_FILTER_WR_FLT_ADDED) {
4981                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
4982                         f->pending = 0;  /* asynchronous setup completed */
4983                         f->valid = 1;
4984                 } else {
4985                         if (rc != FW_FILTER_WR_FLT_DELETED) {
4986                                 /* Add or delete failed, display an error */
4987                                 log(LOG_ERR,
4988                                     "filter %u setup failed with error %u\n",
4989                                     idx, rc);
4990                         }
4991
4992                         clear_filter(f);
4993                         sc->tids.ftids_in_use--;
4994                 }
4995                 ADAPTER_UNLOCK(sc);
4996         }
4997
4998         return (0);
4999 }
5000
5001 static int
5002 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
5003 {
5004         int rc = EINVAL;
5005
5006         if (cntxt->cid > M_CTXTQID)
5007                 return (rc);
5008
5009         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
5010             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
5011                 return (rc);
5012
5013         if (sc->flags & FW_OK) {
5014                 ADAPTER_LOCK(sc);       /* Avoid parallel t4_wr_mbox */
5015                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
5016                     &cntxt->data[0]);
5017                 ADAPTER_UNLOCK(sc);
5018         }
5019
5020         if (rc != 0) {
5021                 /* Read via firmware failed or wasn't even attempted */
5022
5023                 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
5024                     &cntxt->data[0]);
5025         }
5026
5027         return (rc);
5028 }
5029
5030 static int
5031 read_card_mem(struct adapter *sc, struct t4_mem_range *mr)
5032 {
5033         uint32_t base, size, lo, hi, win, off, remaining, i, n;
5034         uint32_t *buf, *b;
5035         int rc;
5036
5037         /* reads are in multiples of 32 bits */
5038         if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
5039                 return (EINVAL);
5040
5041         /*
5042          * We don't want to deal with potential holes so we mandate that the
5043          * requested region must lie entirely within one of the 3 memories.
5044          */
5045         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5046         if (lo & F_EDRAM0_ENABLE) {
5047                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5048                 base = G_EDRAM0_BASE(hi) << 20;
5049                 size = G_EDRAM0_SIZE(hi) << 20;
5050                 if (size > 0 &&
5051                     mr->addr >= base && mr->addr < base + size &&
5052                     mr->addr + mr->len <= base + size)
5053                         goto proceed;
5054         }
5055         if (lo & F_EDRAM1_ENABLE) {
5056                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5057                 base = G_EDRAM1_BASE(hi) << 20;
5058                 size = G_EDRAM1_SIZE(hi) << 20;
5059                 if (size > 0 &&
5060                     mr->addr >= base && mr->addr < base + size &&
5061                     mr->addr + mr->len <= base + size)
5062                         goto proceed;
5063         }
5064         if (lo & F_EXT_MEM_ENABLE) {
5065                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5066                 base = G_EXT_MEM_BASE(hi) << 20;
5067                 size = G_EXT_MEM_SIZE(hi) << 20;
5068                 if (size > 0 &&
5069                     mr->addr >= base && mr->addr < base + size &&
5070                     mr->addr + mr->len <= base + size)
5071                         goto proceed;
5072         }
5073         return (ENXIO);
5074
5075 proceed:
5076         buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5077
5078         /*
5079          * Position the PCIe window (we use memwin2) to the 16B aligned area
5080          * just at/before the requested region.
5081          */
5082         win = mr->addr & ~0xf;
5083         off = mr->addr - win;  /* offset of the requested region in the win */
5084         remaining = mr->len;
5085
5086         while (remaining) {
5087                 t4_write_reg(sc,
5088                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5089                 t4_read_reg(sc,
5090                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5091
5092                 /* number of bytes that we'll copy in the inner loop */
5093                 n = min(remaining, MEMWIN2_APERTURE - off);
5094
5095                 for (i = 0; i < n; i += 4, remaining -= 4)
5096                         *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5097
5098                 win += MEMWIN2_APERTURE;
5099                 off = 0;
5100         }
5101
5102         rc = copyout(buf, mr->data, mr->len);
5103         free(buf, M_CXGBE);
5104
5105         return (rc);
5106 }
5107
5108 int
5109 t4_os_find_pci_capability(struct adapter *sc, int cap)
5110 {
5111         int i;
5112
5113         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5114 }
5115
5116 int
5117 t4_os_pci_save_state(struct adapter *sc)
5118 {
5119         device_t dev;
5120         struct pci_devinfo *dinfo;
5121
5122         dev = sc->dev;
5123         dinfo = device_get_ivars(dev);
5124
5125         pci_cfg_save(dev, dinfo, 0);
5126         return (0);
5127 }
5128
5129 int
5130 t4_os_pci_restore_state(struct adapter *sc)
5131 {
5132         device_t dev;
5133         struct pci_devinfo *dinfo;
5134
5135         dev = sc->dev;
5136         dinfo = device_get_ivars(dev);
5137
5138         pci_cfg_restore(dev, dinfo);
5139         return (0);
5140 }
5141
5142 void
5143 t4_os_portmod_changed(const struct adapter *sc, int idx)
5144 {
5145         struct port_info *pi = sc->port[idx];
5146         static const char *mod_str[] = {
5147                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5148         };
5149
5150         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5151                 if_printf(pi->ifp, "transceiver unplugged.\n");
5152         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5153                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
5154         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5155                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5156         else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) {
5157                 if_printf(pi->ifp, "%s transceiver inserted.\n",
5158                     mod_str[pi->mod_type]);
5159         } else {
5160                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5161                     pi->mod_type);
5162         }
5163 }
5164
5165 void
5166 t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5167 {
5168         struct port_info *pi = sc->port[idx];
5169         struct ifnet *ifp = pi->ifp;
5170
5171         if (link_stat) {
5172                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5173                 if_link_state_change(ifp, LINK_STATE_UP);
5174         } else
5175                 if_link_state_change(ifp, LINK_STATE_DOWN);
5176 }
5177
5178 void
5179 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
5180 {
5181         struct adapter *sc;
5182
5183         mtx_lock(&t4_list_lock);
5184         SLIST_FOREACH(sc, &t4_list, link) {
5185                 /*
5186                  * func should not make any assumptions about what state sc is
5187                  * in - the only guarantee is that sc->sc_lock is a valid lock.
5188                  */
5189                 func(sc, arg);
5190         }
5191         mtx_unlock(&t4_list_lock);
5192 }
5193
5194 static int
5195 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
5196 {
5197        return (0);
5198 }
5199
5200 static int
5201 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
5202 {
5203        return (0);
5204 }
5205
5206 static int
5207 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5208     struct thread *td)
5209 {
5210         int rc;
5211         struct adapter *sc = dev->si_drv1;
5212
5213         rc = priv_check(td, PRIV_DRIVER);
5214         if (rc != 0)
5215                 return (rc);
5216
5217         switch (cmd) {
5218         case CHELSIO_T4_GETREG: {
5219                 struct t4_reg *edata = (struct t4_reg *)data;
5220
5221                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5222                         return (EFAULT);
5223
5224                 if (edata->size == 4)
5225                         edata->val = t4_read_reg(sc, edata->addr);
5226                 else if (edata->size == 8)
5227                         edata->val = t4_read_reg64(sc, edata->addr);
5228                 else
5229                         return (EINVAL);
5230
5231                 break;
5232         }
5233         case CHELSIO_T4_SETREG: {
5234                 struct t4_reg *edata = (struct t4_reg *)data;
5235
5236                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5237                         return (EFAULT);
5238
5239                 if (edata->size == 4) {
5240                         if (edata->val & 0xffffffff00000000)
5241                                 return (EINVAL);
5242                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5243                 } else if (edata->size == 8)
5244                         t4_write_reg64(sc, edata->addr, edata->val);
5245                 else
5246                         return (EINVAL);
5247                 break;
5248         }
5249         case CHELSIO_T4_REGDUMP: {
5250                 struct t4_regdump *regs = (struct t4_regdump *)data;
5251                 int reglen = T4_REGDUMP_SIZE;
5252                 uint8_t *buf;
5253
5254                 if (regs->len < reglen) {
5255                         regs->len = reglen; /* hint to the caller */
5256                         return (ENOBUFS);
5257                 }
5258
5259                 regs->len = reglen;
5260                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5261                 t4_get_regs(sc, regs, buf);
5262                 rc = copyout(buf, regs->data, reglen);
5263                 free(buf, M_CXGBE);
5264                 break;
5265         }
5266         case CHELSIO_T4_GET_FILTER_MODE:
5267                 rc = get_filter_mode(sc, (uint32_t *)data);
5268                 break;
5269         case CHELSIO_T4_SET_FILTER_MODE:
5270                 rc = set_filter_mode(sc, *(uint32_t *)data);
5271                 break;
5272         case CHELSIO_T4_GET_FILTER:
5273                 ADAPTER_LOCK(sc);
5274                 rc = get_filter(sc, (struct t4_filter *)data);
5275                 ADAPTER_UNLOCK(sc);
5276                 break;
5277         case CHELSIO_T4_SET_FILTER:
5278                 ADAPTER_LOCK(sc);
5279                 rc = set_filter(sc, (struct t4_filter *)data);
5280                 ADAPTER_UNLOCK(sc);
5281                 break;
5282         case CHELSIO_T4_DEL_FILTER:
5283                 ADAPTER_LOCK(sc);
5284                 rc = del_filter(sc, (struct t4_filter *)data);
5285                 ADAPTER_UNLOCK(sc);
5286                 break;
5287         case CHELSIO_T4_GET_SGE_CONTEXT:
5288                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
5289                 break;
5290         case CHELSIO_T4_LOAD_FW: {
5291                 struct t4_data *fw = (struct t4_data *)data;
5292                 uint8_t *fw_data;
5293
5294                 if (sc->flags & FULL_INIT_DONE)
5295                         return (EBUSY);
5296
5297                 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT);
5298                 if (fw_data == NULL)
5299                         return (ENOMEM);
5300
5301                 rc = copyin(fw->data, fw_data, fw->len);
5302                 if (rc == 0)
5303                         rc = -t4_load_fw(sc, fw_data, fw->len);
5304
5305                 free(fw_data, M_CXGBE);
5306                 break;
5307         }
5308         case CHELSIO_T4_GET_MEM:
5309                 rc = read_card_mem(sc, (struct t4_mem_range *)data);
5310                 break;
5311         default:
5312                 rc = EINVAL;
5313         }
5314
5315         return (rc);
5316 }
5317
5318 #ifdef TCP_OFFLOAD
5319 static int
5320 toe_capability(struct port_info *pi, int enable)
5321 {
5322         int rc;
5323         struct adapter *sc = pi->adapter;
5324
5325         ADAPTER_LOCK_ASSERT_OWNED(sc);
5326
5327         if (!is_offload(sc))
5328                 return (ENODEV);
5329
5330         if (enable) {
5331                 if (!(sc->flags & FULL_INIT_DONE)) {
5332                         log(LOG_WARNING,
5333                             "You must enable a cxgbe interface first\n");
5334                         return (EAGAIN);
5335                 }
5336
5337                 if (isset(&sc->offload_map, pi->port_id))
5338                         return (0);
5339
5340                 if (!(sc->flags & TOM_INIT_DONE)) {
5341                         rc = t4_activate_uld(sc, ULD_TOM);
5342                         if (rc == EAGAIN) {
5343                                 log(LOG_WARNING,
5344                                     "You must kldload t4_tom.ko before trying "
5345                                     "to enable TOE on a cxgbe interface.\n");
5346                         }
5347                         if (rc != 0)
5348                                 return (rc);
5349                         KASSERT(sc->tom_softc != NULL,
5350                             ("%s: TOM activated but softc NULL", __func__));
5351                         KASSERT(sc->flags & TOM_INIT_DONE,
5352                             ("%s: TOM activated but flag not set", __func__));
5353                 }
5354
5355                 setbit(&sc->offload_map, pi->port_id);
5356         } else {
5357                 if (!isset(&sc->offload_map, pi->port_id))
5358                         return (0);
5359
5360                 KASSERT(sc->flags & TOM_INIT_DONE,
5361                     ("%s: TOM never initialized?", __func__));
5362                 clrbit(&sc->offload_map, pi->port_id);
5363         }
5364
5365         return (0);
5366 }
5367
5368 /*
5369  * Add an upper layer driver to the global list.
5370  */
5371 int
5372 t4_register_uld(struct uld_info *ui)
5373 {
5374         int rc = 0;
5375         struct uld_info *u;
5376
5377         mtx_lock(&t4_uld_list_lock);
5378         SLIST_FOREACH(u, &t4_uld_list, link) {
5379             if (u->uld_id == ui->uld_id) {
5380                     rc = EEXIST;
5381                     goto done;
5382             }
5383         }
5384
5385         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5386         ui->refcount = 0;
5387 done:
5388         mtx_unlock(&t4_uld_list_lock);
5389         return (rc);
5390 }
5391
5392 int
5393 t4_unregister_uld(struct uld_info *ui)
5394 {
5395         int rc = EINVAL;
5396         struct uld_info *u;
5397
5398         mtx_lock(&t4_uld_list_lock);
5399
5400         SLIST_FOREACH(u, &t4_uld_list, link) {
5401             if (u == ui) {
5402                     if (ui->refcount > 0) {
5403                             rc = EBUSY;
5404                             goto done;
5405                     }
5406
5407                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5408                     rc = 0;
5409                     goto done;
5410             }
5411         }
5412 done:
5413         mtx_unlock(&t4_uld_list_lock);
5414         return (rc);
5415 }
5416
5417 int
5418 t4_activate_uld(struct adapter *sc, int id)
5419 {
5420         int rc = EAGAIN;
5421         struct uld_info *ui;
5422
5423         mtx_lock(&t4_uld_list_lock);
5424
5425         SLIST_FOREACH(ui, &t4_uld_list, link) {
5426                 if (ui->uld_id == id) {
5427                         rc = ui->activate(sc);
5428                         if (rc == 0)
5429                                 ui->refcount++;
5430                         goto done;
5431                 }
5432         }
5433 done:
5434         mtx_unlock(&t4_uld_list_lock);
5435
5436         return (rc);
5437 }
5438
5439 int
5440 t4_deactivate_uld(struct adapter *sc, int id)
5441 {
5442         int rc = EINVAL;
5443         struct uld_info *ui;
5444
5445         mtx_lock(&t4_uld_list_lock);
5446
5447         SLIST_FOREACH(ui, &t4_uld_list, link) {
5448                 if (ui->uld_id == id) {
5449                         rc = ui->deactivate(sc);
5450                         if (rc == 0)
5451                                 ui->refcount--;
5452                         goto done;
5453                 }
5454         }
5455 done:
5456         mtx_unlock(&t4_uld_list_lock);
5457
5458         return (rc);
5459 }
5460 #endif
5461
5462 /*
5463  * Come up with reasonable defaults for some of the tunables, provided they're
5464  * not set by the user (in which case we'll use the values as is).
5465  */
5466 static void
5467 tweak_tunables(void)
5468 {
5469         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
5470
5471         if (t4_ntxq10g < 1)
5472                 t4_ntxq10g = min(nc, NTXQ_10G);
5473
5474         if (t4_ntxq1g < 1)
5475                 t4_ntxq1g = min(nc, NTXQ_1G);
5476
5477         if (t4_nrxq10g < 1)
5478                 t4_nrxq10g = min(nc, NRXQ_10G);
5479
5480         if (t4_nrxq1g < 1)
5481                 t4_nrxq1g = min(nc, NRXQ_1G);
5482
5483 #ifdef TCP_OFFLOAD
5484         if (t4_nofldtxq10g < 1)
5485                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
5486
5487         if (t4_nofldtxq1g < 1)
5488                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
5489
5490         if (t4_nofldrxq10g < 1)
5491                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
5492
5493         if (t4_nofldrxq1g < 1)
5494                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
5495
5496         if (t4_toecaps_allowed == -1)
5497                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
5498 #else
5499         if (t4_toecaps_allowed == -1)
5500                 t4_toecaps_allowed = 0;
5501 #endif
5502
5503         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
5504                 t4_tmr_idx_10g = TMR_IDX_10G;
5505
5506         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
5507                 t4_pktc_idx_10g = PKTC_IDX_10G;
5508
5509         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
5510                 t4_tmr_idx_1g = TMR_IDX_1G;
5511
5512         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
5513                 t4_pktc_idx_1g = PKTC_IDX_1G;
5514
5515         if (t4_qsize_txq < 128)
5516                 t4_qsize_txq = 128;
5517
5518         if (t4_qsize_rxq < 128)
5519                 t4_qsize_rxq = 128;
5520         while (t4_qsize_rxq & 7)
5521                 t4_qsize_rxq++;
5522
5523         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
5524 }
5525
5526 static int
5527 t4_mod_event(module_t mod, int cmd, void *arg)
5528 {
5529         int rc = 0;
5530
5531         switch (cmd) {
5532         case MOD_LOAD:
5533                 t4_sge_modload();
5534                 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
5535                 SLIST_INIT(&t4_list);
5536 #ifdef TCP_OFFLOAD
5537                 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
5538                 SLIST_INIT(&t4_uld_list);
5539 #endif
5540                 tweak_tunables();
5541                 break;
5542
5543         case MOD_UNLOAD:
5544 #ifdef TCP_OFFLOAD
5545                 mtx_lock(&t4_uld_list_lock);
5546                 if (!SLIST_EMPTY(&t4_uld_list)) {
5547                         rc = EBUSY;
5548                         mtx_unlock(&t4_uld_list_lock);
5549                         break;
5550                 }
5551                 mtx_unlock(&t4_uld_list_lock);
5552                 mtx_destroy(&t4_uld_list_lock);
5553 #endif
5554                 mtx_lock(&t4_list_lock);
5555                 if (!SLIST_EMPTY(&t4_list)) {
5556                         rc = EBUSY;
5557                         mtx_unlock(&t4_list_lock);
5558                         break;
5559                 }
5560                 mtx_unlock(&t4_list_lock);
5561                 mtx_destroy(&t4_list_lock);
5562                 break;
5563         }
5564
5565         return (rc);
5566 }
5567
5568 static devclass_t t4_devclass;
5569 static devclass_t cxgbe_devclass;
5570
5571 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
5572 MODULE_VERSION(t4nex, 1);
5573
5574 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
5575 MODULE_VERSION(cxgbe, 1);