sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58
  59 #include "common/common.h"
  60 #include "common/t4_msg.h"
  61 #include "common/t4_regs.h"
  62 #include "common/t4_regs_values.h"
  63 #include "t4_ioctl.h"
  64 #include "t4_l2t.h"
  65
  66 /* T4 bus driver interface */
  67 static int t4_probe(device_t);
  68 static int t4_attach(device_t);
  69 static int t4_detach(device_t);
  70 static device_method_t t4_methods[] = {
  71         DEVMETHOD(device_probe,         t4_probe),
  72         DEVMETHOD(device_attach,        t4_attach),
  73         DEVMETHOD(device_detach,        t4_detach),
  74
  75         DEVMETHOD_END
  76 };
  77 static driver_t t4_driver = {
  78         "t4nex",
  79         t4_methods,
  80         sizeof(struct adapter)
  81 };
  82
  83
  84 /* T4 port (cxgbe) interface */
  85 static int cxgbe_probe(device_t);
  86 static int cxgbe_attach(device_t);
  87 static int cxgbe_detach(device_t);
  88 static device_method_t cxgbe_methods[] = {
  89         DEVMETHOD(device_probe,         cxgbe_probe),
  90         DEVMETHOD(device_attach,        cxgbe_attach),
  91         DEVMETHOD(device_detach,        cxgbe_detach),
  92         { 0, 0 }
  93 };
  94 static driver_t cxgbe_driver = {
  95         "cxgbe",
  96         cxgbe_methods,
  97         sizeof(struct port_info)
  98 };
  99
 100 static d_ioctl_t t4_ioctl;
 101 static d_open_t t4_open;
 102 static d_close_t t4_close;
 103
 104 static struct cdevsw t4_cdevsw = {
 105        .d_version = D_VERSION,
 106        .d_flags = 0,
 107        .d_open = t4_open,
 108        .d_close = t4_close,
 109        .d_ioctl = t4_ioctl,
 110        .d_name = "t4nex",
 111 };
 112
 113 /* ifnet + media interface */
 114 static void cxgbe_init(void *);
 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 117 static void cxgbe_qflush(struct ifnet *);
 118 static int cxgbe_media_change(struct ifnet *);
 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 120
 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
 122
 123 /*
 124  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 125  * then ADAPTER_LOCK, then t4_uld_list_lock.
 126  */
 127 static struct mtx t4_list_lock;
 128 static SLIST_HEAD(, adapter) t4_list;
 129 #ifdef TCP_OFFLOAD
 130 static struct mtx t4_uld_list_lock;
 131 static SLIST_HEAD(, uld_info) t4_uld_list;
 132 #endif
 133
 134 /*
 135  * Tunables.  See tweak_tunables() too.
 136  */
 137
 138 /*
 139  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 140  */
 141 #define NTXQ_10G 16
 142 static int t4_ntxq10g = -1;
 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 144
 145 #define NRXQ_10G 8
 146 static int t4_nrxq10g = -1;
 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 148
 149 #define NTXQ_1G 4
 150 static int t4_ntxq1g = -1;
 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 152
 153 #define NRXQ_1G 2
 154 static int t4_nrxq1g = -1;
 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 156
 157 #ifdef TCP_OFFLOAD
 158 #define NOFLDTXQ_10G 8
 159 static int t4_nofldtxq10g = -1;
 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 161
 162 #define NOFLDRXQ_10G 2
 163 static int t4_nofldrxq10g = -1;
 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 165
 166 #define NOFLDTXQ_1G 2
 167 static int t4_nofldtxq1g = -1;
 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 169
 170 #define NOFLDRXQ_1G 1
 171 static int t4_nofldrxq1g = -1;
 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 173 #endif
 174
 175 /*
 176  * Holdoff parameters for 10G and 1G ports.
 177  */
 178 #define TMR_IDX_10G 1
 179 static int t4_tmr_idx_10g = TMR_IDX_10G;
 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 181
 182 #define PKTC_IDX_10G (-1)
 183 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 185
 186 #define TMR_IDX_1G 1
 187 static int t4_tmr_idx_1g = TMR_IDX_1G;
 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 189
 190 #define PKTC_IDX_1G (-1)
 191 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 193
 194 /*
 195  * Size (# of entries) of each tx and rx queue.
 196  */
 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 199
 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 202
 203 /*
 204  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 205  */
 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 208
 209 /*
 210  * Configuration file.
 211  */
 212 static char t4_cfg_file[32] = "default";
 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 214
 215 /*
 216  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 217  * encouraged respectively).
 218  */
 219 static unsigned int t4_fw_install = 1;
 220 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 221
 222 /*
 223  * ASIC features that will be used.  Disable the ones you don't want so that the
 224  * chip resources aren't wasted on features that will not be used.
 225  */
 226 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 227 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 228
 229 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 230 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 231
 232 static int t4_toecaps_allowed = -1;
 233 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 234
 235 static int t4_rdmacaps_allowed = 0;
 236 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 237
 238 static int t4_iscsicaps_allowed = 0;
 239 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 240
 241 static int t4_fcoecaps_allowed = 0;
 242 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 243
 244 struct intrs_and_queues {
 245         int intr_type;          /* INTx, MSI, or MSI-X */
 246         int nirq;               /* Number of vectors */
 247         int intr_flags;
 248         int ntxq10g;            /* # of NIC txq's for each 10G port */
 249         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 250         int ntxq1g;             /* # of NIC txq's for each 1G port */
 251         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 252 #ifdef TCP_OFFLOAD
 253         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 254         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 255         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 256         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 257 #endif
 258 };
 259
 260 struct filter_entry {
 261         uint32_t valid:1;       /* filter allocated and valid */
 262         uint32_t locked:1;      /* filter is administratively locked */
 263         uint32_t pending:1;     /* filter action is pending firmware reply */
 264         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 265         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 266
 267         struct t4_filter_specification fs;
 268 };
 269
 270 enum {
 271         XGMAC_MTU       = (1 << 0),
 272         XGMAC_PROMISC   = (1 << 1),
 273         XGMAC_ALLMULTI  = (1 << 2),
 274         XGMAC_VLANEX    = (1 << 3),
 275         XGMAC_UCADDR    = (1 << 4),
 276         XGMAC_MCADDRS   = (1 << 5),
 277
 278         XGMAC_ALL       = 0xffff
 279 };
 280
 281 static int map_bars(struct adapter *);
 282 static void setup_memwin(struct adapter *);
 283 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 284     struct intrs_and_queues *);
 285 static int prep_firmware(struct adapter *);
 286 static int upload_config_file(struct adapter *, const struct firmware *,
 287     uint32_t *, uint32_t *);
 288 static int partition_resources(struct adapter *, const struct firmware *);
 289 static int get_params__pre_init(struct adapter *);
 290 static int get_params__post_init(struct adapter *);
 291 static int set_params__post_init(struct adapter *);
 292 static void t4_set_desc(struct adapter *);
 293 static void build_medialist(struct port_info *);
 294 static int update_mac_settings(struct port_info *, int);
 295 static int cxgbe_init_synchronized(struct port_info *);
 296 static int cxgbe_uninit_synchronized(struct port_info *);
 297 static int setup_intr_handlers(struct adapter *);
 298 static int adapter_full_init(struct adapter *);
 299 static int adapter_full_uninit(struct adapter *);
 300 static int port_full_init(struct port_info *);
 301 static int port_full_uninit(struct port_info *);
 302 static void quiesce_eq(struct adapter *, struct sge_eq *);
 303 static void quiesce_iq(struct adapter *, struct sge_iq *);
 304 static void quiesce_fl(struct adapter *, struct sge_fl *);
 305 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 306     driver_intr_t *, void *, char *);
 307 static int t4_free_irq(struct adapter *, struct irq *);
 308 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 309     unsigned int);
 310 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 311 static void cxgbe_tick(void *);
 312 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 313 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 314     struct mbuf *);
 315 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 316 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 317 static int t4_sysctls(struct adapter *);
 318 static int cxgbe_sysctls(struct port_info *);
 319 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 320 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 321 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 322 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 323 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 324 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 325 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 326 #ifdef SBUF_DRAIN
 327 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 328 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 329 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 330 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 331 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 332 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 333 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 334 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 335 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 336 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 337 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 338 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 339 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 340 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 341 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 342 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 343 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 344 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 345 #endif
 346 static inline void txq_start(struct ifnet *, struct sge_txq *);
 347 static uint32_t fconf_to_mode(uint32_t);
 348 static uint32_t mode_to_fconf(uint32_t);
 349 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 350 static int get_filter_mode(struct adapter *, uint32_t *);
 351 static int set_filter_mode(struct adapter *, uint32_t);
 352 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 353 static int get_filter(struct adapter *, struct t4_filter *);
 354 static int set_filter(struct adapter *, struct t4_filter *);
 355 static int del_filter(struct adapter *, struct t4_filter *);
 356 static void clear_filter(struct filter_entry *);
 357 static int set_filter_wr(struct adapter *, int);
 358 static int del_filter_wr(struct adapter *, int);
 359 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 360 static int load_fw(struct adapter *, struct t4_data *);
 361 static int read_card_mem(struct adapter *, struct t4_mem_range *);
 362 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 363 #ifdef TCP_OFFLOAD
 364 static int toe_capability(struct port_info *, int);
 365 #endif
 366 static int t4_mod_event(module_t, int, void *);
 367
 368 struct t4_pciids {
 369         uint16_t device;
 370         char *desc;
 371 } t4_pciids[] = {
 372         {0xa000, "Chelsio Terminator 4 FPGA"},
 373         {0x4400, "Chelsio T440-dbg"},
 374         {0x4401, "Chelsio T420-CR"},
 375         {0x4402, "Chelsio T422-CR"},
 376         {0x4403, "Chelsio T440-CR"},
 377         {0x4404, "Chelsio T420-BCH"},
 378         {0x4405, "Chelsio T440-BCH"},
 379         {0x4406, "Chelsio T440-CH"},
 380         {0x4407, "Chelsio T420-SO"},
 381         {0x4408, "Chelsio T420-CX"},
 382         {0x4409, "Chelsio T420-BT"},
 383         {0x440a, "Chelsio T404-BT"},
 384         {0x440e, "Chelsio T440-LP-CR"},
 385 };
 386
 387 #ifdef TCP_OFFLOAD
 388 /*
 389  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 390  * exactly the same for both rxq and ofld_rxq.
 391  */
 392 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 393 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 394 #endif
 395
 396 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 397 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 398 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 399
 400 static int
 401 t4_probe(device_t dev)
 402 {
 403         int i;
 404         uint16_t v = pci_get_vendor(dev);
 405         uint16_t d = pci_get_device(dev);
 406         uint8_t f = pci_get_function(dev);
 407
 408         if (v != PCI_VENDOR_ID_CHELSIO)
 409                 return (ENXIO);
 410
 411         /* Attach only to PF0 of the FPGA */
 412         if (d == 0xa000 && f != 0)
 413                 return (ENXIO);
 414
 415         for (i = 0; i < nitems(t4_pciids); i++) {
 416                 if (d == t4_pciids[i].device) {
 417                         device_set_desc(dev, t4_pciids[i].desc);
 418                         return (BUS_PROBE_DEFAULT);
 419                 }
 420         }
 421
 422         return (ENXIO);
 423 }
 424
 425 static int
 426 t4_attach(device_t dev)
 427 {
 428         struct adapter *sc;
 429         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 430         struct intrs_and_queues iaq;
 431         struct sge *s;
 432 #ifdef TCP_OFFLOAD
 433         int ofld_rqidx, ofld_tqidx;
 434 #endif
 435
 436         sc = device_get_softc(dev);
 437         sc->dev = dev;
 438
 439         pci_enable_busmaster(dev);
 440         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 441                 uint32_t v;
 442
 443                 pci_set_max_read_req(dev, 4096);
 444                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 445                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 446                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 447         }
 448
 449         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 450             device_get_nameunit(dev));
 451         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 452         mtx_lock(&t4_list_lock);
 453         SLIST_INSERT_HEAD(&t4_list, sc, link);
 454         mtx_unlock(&t4_list_lock);
 455
 456         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 457         TAILQ_INIT(&sc->sfl);
 458         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 459
 460         rc = map_bars(sc);
 461         if (rc != 0)
 462                 goto done; /* error message displayed already */
 463
 464         /*
 465          * This is the real PF# to which we're attaching.  Works from within PCI
 466          * passthrough environments too, where pci_get_function() could return a
 467          * different PF# depending on the passthrough configuration.  We need to
 468          * use the real PF# in all our communication with the firmware.
 469          */
 470         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 471         sc->mbox = sc->pf;
 472
 473         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 474         sc->an_handler = an_not_handled;
 475         for (i = 0; i < nitems(sc->cpl_handler); i++)
 476                 sc->cpl_handler[i] = cpl_not_handled;
 477         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 478                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 479         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 480
 481         /* Prepare the adapter for operation */
 482         rc = -t4_prep_adapter(sc);
 483         if (rc != 0) {
 484                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 485                 goto done;
 486         }
 487
 488         /*
 489          * Do this really early, with the memory windows set up even before the
 490          * character device.  The userland tool's register i/o and mem read
 491          * will work even in "recovery mode".
 492          */
 493         setup_memwin(sc);
 494         sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
 495             GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
 496         sc->cdev->si_drv1 = sc;
 497
 498         /* Go no further if recovery mode has been requested. */
 499         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 500                 device_printf(dev, "recovery mode.\n");
 501                 goto done;
 502         }
 503
 504         /* Prepare the firmware for operation */
 505         rc = prep_firmware(sc);
 506         if (rc != 0)
 507                 goto done; /* error message displayed already */
 508
 509         rc = get_params__pre_init(sc);
 510         if (rc != 0)
 511                 goto done; /* error message displayed already */
 512
 513         rc = t4_sge_init(sc);
 514         if (rc != 0)
 515                 goto done; /* error message displayed already */
 516
 517         if (sc->flags & MASTER_PF) {
 518                 /* get basic stuff going */
 519                 rc = -t4_fw_initialize(sc, sc->mbox);
 520                 if (rc != 0) {
 521                         device_printf(dev, "early init failed: %d.\n", rc);
 522                         goto done;
 523                 }
 524         }
 525
 526         rc = get_params__post_init(sc);
 527         if (rc != 0)
 528                 goto done; /* error message displayed already */
 529
 530         rc = set_params__post_init(sc);
 531         if (rc != 0)
 532                 goto done; /* error message displayed already */
 533
 534         if (sc->flags & MASTER_PF) {
 535                 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
 536
 537                 /* final tweaks to some settings */
 538
 539                 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
 540                     sc->params.b_wnd);
 541                 /* 4K, 16K, 64K, 256K DDP "page sizes" */
 542                 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) |
 543                     V_HPZ2(4) | V_HPZ3(6));
 544                 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB);
 545                 t4_set_reg_field(sc, A_TP_PARA_REG5,
 546                     V_INDICATESIZE(M_INDICATESIZE) |
 547                     F_REARMDDPOFFSET | F_RESETDDPOFFSET,
 548                     V_INDICATESIZE(indsz) |
 549                     F_REARMDDPOFFSET | F_RESETDDPOFFSET);
 550         } else {
 551                 /*
 552                  * XXX: Verify that we can live with whatever the master driver
 553                  * has done so far, and hope that it doesn't change any global
 554                  * setting from underneath us in the future.
 555                  */
 556         }
 557
 558         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
 559             A_TP_VLAN_PRI_MAP);
 560
 561         for (i = 0; i < NCHAN; i++)
 562                 sc->params.tp.tx_modq[i] = i;
 563
 564         rc = t4_create_dma_tag(sc);
 565         if (rc != 0)
 566                 goto done; /* error message displayed already */
 567
 568         /*
 569          * First pass over all the ports - allocate VIs and initialize some
 570          * basic parameters like mac address, port type, etc.  We also figure
 571          * out whether a port is 10G or 1G and use that information when
 572          * calculating how many interrupts to attempt to allocate.
 573          */
 574         n10g = n1g = 0;
 575         for_each_port(sc, i) {
 576                 struct port_info *pi;
 577
 578                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 579                 sc->port[i] = pi;
 580
 581                 /* These must be set before t4_port_init */
 582                 pi->adapter = sc;
 583                 pi->port_id = i;
 584
 585                 /* Allocate the vi and initialize parameters like mac addr */
 586                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 587                 if (rc != 0) {
 588                         device_printf(dev, "unable to initialize port %d: %d\n",
 589                             i, rc);
 590                         free(pi, M_CXGBE);
 591                         sc->port[i] = NULL;
 592                         goto done;
 593                 }
 594
 595                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 596                     device_get_nameunit(dev), i);
 597                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 598
 599                 if (is_10G_port(pi)) {
 600                         n10g++;
 601                         pi->tmr_idx = t4_tmr_idx_10g;
 602                         pi->pktc_idx = t4_pktc_idx_10g;
 603                 } else {
 604                         n1g++;
 605                         pi->tmr_idx = t4_tmr_idx_1g;
 606                         pi->pktc_idx = t4_pktc_idx_1g;
 607                 }
 608
 609                 pi->xact_addr_filt = -1;
 610
 611                 pi->qsize_rxq = t4_qsize_rxq;
 612                 pi->qsize_txq = t4_qsize_txq;
 613
 614                 pi->dev = device_add_child(dev, "cxgbe", -1);
 615                 if (pi->dev == NULL) {
 616                         device_printf(dev,
 617                             "failed to add device for port %d.\n", i);
 618                         rc = ENXIO;
 619                         goto done;
 620                 }
 621                 device_set_softc(pi->dev, pi);
 622         }
 623
 624         /*
 625          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 626          */
 627         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 628         if (rc != 0)
 629                 goto done; /* error message displayed already */
 630
 631         sc->intr_type = iaq.intr_type;
 632         sc->intr_count = iaq.nirq;
 633         sc->flags |= iaq.intr_flags;
 634
 635         s = &sc->sge;
 636         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 637         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 638         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 639         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 640         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 641
 642 #ifdef TCP_OFFLOAD
 643         if (is_offload(sc)) {
 644
 645                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 646                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 647                 s->neq += s->nofldtxq + s->nofldrxq;
 648                 s->niq += s->nofldrxq;
 649
 650                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 651                     M_CXGBE, M_ZERO | M_WAITOK);
 652                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 653                     M_CXGBE, M_ZERO | M_WAITOK);
 654         }
 655 #endif
 656
 657         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 658             M_ZERO | M_WAITOK);
 659         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 660             M_ZERO | M_WAITOK);
 661         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 662             M_ZERO | M_WAITOK);
 663         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 664             M_ZERO | M_WAITOK);
 665         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 666             M_ZERO | M_WAITOK);
 667
 668         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 669             M_ZERO | M_WAITOK);
 670
 671         t4_init_l2t(sc, M_WAITOK);
 672
 673         /*
 674          * Second pass over the ports.  This time we know the number of rx and
 675          * tx queues that each port should get.
 676          */
 677         rqidx = tqidx = 0;
 678 #ifdef TCP_OFFLOAD
 679         ofld_rqidx = ofld_tqidx = 0;
 680 #endif
 681         for_each_port(sc, i) {
 682                 struct port_info *pi = sc->port[i];
 683
 684                 if (pi == NULL)
 685                         continue;
 686
 687                 pi->first_rxq = rqidx;
 688                 pi->first_txq = tqidx;
 689                 if (is_10G_port(pi)) {
 690                         pi->nrxq = iaq.nrxq10g;
 691                         pi->ntxq = iaq.ntxq10g;
 692                 } else {
 693                         pi->nrxq = iaq.nrxq1g;
 694                         pi->ntxq = iaq.ntxq1g;
 695                 }
 696
 697                 rqidx += pi->nrxq;
 698                 tqidx += pi->ntxq;
 699
 700 #ifdef TCP_OFFLOAD
 701                 if (is_offload(sc)) {
 702                         pi->first_ofld_rxq = ofld_rqidx;
 703                         pi->first_ofld_txq = ofld_tqidx;
 704                         if (is_10G_port(pi)) {
 705                                 pi->nofldrxq = iaq.nofldrxq10g;
 706                                 pi->nofldtxq = iaq.nofldtxq10g;
 707                         } else {
 708                                 pi->nofldrxq = iaq.nofldrxq1g;
 709                                 pi->nofldtxq = iaq.nofldtxq1g;
 710                         }
 711                         ofld_rqidx += pi->nofldrxq;
 712                         ofld_tqidx += pi->nofldtxq;
 713                 }
 714 #endif
 715         }
 716
 717         rc = setup_intr_handlers(sc);
 718         if (rc != 0) {
 719                 device_printf(dev,
 720                     "failed to setup interrupt handlers: %d\n", rc);
 721                 goto done;
 722         }
 723
 724         rc = bus_generic_attach(dev);
 725         if (rc != 0) {
 726                 device_printf(dev,
 727                     "failed to attach all child ports: %d\n", rc);
 728                 goto done;
 729         }
 730
 731         device_printf(dev,
 732             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 733             sc->params.pci.width, sc->params.nports, sc->intr_count,
 734             sc->intr_type == INTR_MSIX ? "MSI-X" :
 735             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 736             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 737
 738         t4_set_desc(sc);
 739
 740 done:
 741         if (rc != 0 && sc->cdev) {
 742                 /* cdev was created and so cxgbetool works; recover that way. */
 743                 device_printf(dev,
 744                     "error during attach, adapter is now in recovery mode.\n");
 745                 rc = 0;
 746         }
 747
 748         if (rc != 0)
 749                 t4_detach(dev);
 750         else
 751                 t4_sysctls(sc);
 752
 753         return (rc);
 754 }
 755
 756 /*
 757  * Idempotent
 758  */
 759 static int
 760 t4_detach(device_t dev)
 761 {
 762         struct adapter *sc;
 763         struct port_info *pi;
 764         int i, rc;
 765
 766         sc = device_get_softc(dev);
 767
 768         if (sc->flags & FULL_INIT_DONE)
 769                 t4_intr_disable(sc);
 770
 771         if (sc->cdev) {
 772                 destroy_dev(sc->cdev);
 773                 sc->cdev = NULL;
 774         }
 775
 776         rc = bus_generic_detach(dev);
 777         if (rc) {
 778                 device_printf(dev,
 779                     "failed to detach child devices: %d\n", rc);
 780                 return (rc);
 781         }
 782
 783         for (i = 0; i < sc->intr_count; i++)
 784                 t4_free_irq(sc, &sc->irq[i]);
 785
 786         for (i = 0; i < MAX_NPORTS; i++) {
 787                 pi = sc->port[i];
 788                 if (pi) {
 789                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 790                         if (pi->dev)
 791                                 device_delete_child(dev, pi->dev);
 792
 793                         mtx_destroy(&pi->pi_lock);
 794                         free(pi, M_CXGBE);
 795                 }
 796         }
 797
 798         if (sc->flags & FULL_INIT_DONE)
 799                 adapter_full_uninit(sc);
 800
 801         if (sc->flags & FW_OK)
 802                 t4_fw_bye(sc, sc->mbox);
 803
 804         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 805                 pci_release_msi(dev);
 806
 807         if (sc->regs_res)
 808                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 809                     sc->regs_res);
 810
 811         if (sc->msix_res)
 812                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 813                     sc->msix_res);
 814
 815         if (sc->l2t)
 816                 t4_free_l2t(sc->l2t);
 817
 818 #ifdef TCP_OFFLOAD
 819         free(sc->sge.ofld_rxq, M_CXGBE);
 820         free(sc->sge.ofld_txq, M_CXGBE);
 821 #endif
 822         free(sc->irq, M_CXGBE);
 823         free(sc->sge.rxq, M_CXGBE);
 824         free(sc->sge.txq, M_CXGBE);
 825         free(sc->sge.ctrlq, M_CXGBE);
 826         free(sc->sge.iqmap, M_CXGBE);
 827         free(sc->sge.eqmap, M_CXGBE);
 828         free(sc->tids.ftid_tab, M_CXGBE);
 829         t4_destroy_dma_tag(sc);
 830         if (mtx_initialized(&sc->sc_lock)) {
 831                 mtx_lock(&t4_list_lock);
 832                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 833                 mtx_unlock(&t4_list_lock);
 834                 mtx_destroy(&sc->sc_lock);
 835         }
 836
 837         if (mtx_initialized(&sc->tids.ftid_lock))
 838                 mtx_destroy(&sc->tids.ftid_lock);
 839         if (mtx_initialized(&sc->sfl_lock))
 840                 mtx_destroy(&sc->sfl_lock);
 841
 842         bzero(sc, sizeof(*sc));
 843
 844         return (0);
 845 }
 846
 847
 848 static int
 849 cxgbe_probe(device_t dev)
 850 {
 851         char buf[128];
 852         struct port_info *pi = device_get_softc(dev);
 853
 854         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 855         device_set_desc_copy(dev, buf);
 856
 857         return (BUS_PROBE_DEFAULT);
 858 }
 859
 860 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 861     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 862     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
 863 #define T4_CAP_ENABLE (T4_CAP)
 864
 865 static int
 866 cxgbe_attach(device_t dev)
 867 {
 868         struct port_info *pi = device_get_softc(dev);
 869         struct ifnet *ifp;
 870
 871         /* Allocate an ifnet and set it up */
 872         ifp = if_alloc(IFT_ETHER);
 873         if (ifp == NULL) {
 874                 device_printf(dev, "Cannot allocate ifnet\n");
 875                 return (ENOMEM);
 876         }
 877         pi->ifp = ifp;
 878         ifp->if_softc = pi;
 879
 880         callout_init(&pi->tick, CALLOUT_MPSAFE);
 881
 882         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 883         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 884
 885         ifp->if_init = cxgbe_init;
 886         ifp->if_ioctl = cxgbe_ioctl;
 887         ifp->if_transmit = cxgbe_transmit;
 888         ifp->if_qflush = cxgbe_qflush;
 889
 890         ifp->if_capabilities = T4_CAP;
 891 #ifdef TCP_OFFLOAD
 892         if (is_offload(pi->adapter))
 893                 ifp->if_capabilities |= IFCAP_TOE;
 894 #endif
 895         ifp->if_capenable = T4_CAP_ENABLE;
 896         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 897             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 898
 899         /* Initialize ifmedia for this port */
 900         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 901             cxgbe_media_status);
 902         build_medialist(pi);
 903
 904         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 905             EVENTHANDLER_PRI_ANY);
 906
 907         ether_ifattach(ifp, pi->hw_addr);
 908
 909 #ifdef TCP_OFFLOAD
 910         if (is_offload(pi->adapter)) {
 911                 device_printf(dev,
 912                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
 913                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
 914         } else
 915 #endif
 916                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
 917
 918         cxgbe_sysctls(pi);
 919
 920         return (0);
 921 }
 922
 923 static int
 924 cxgbe_detach(device_t dev)
 925 {
 926         struct port_info *pi = device_get_softc(dev);
 927         struct adapter *sc = pi->adapter;
 928         struct ifnet *ifp = pi->ifp;
 929
 930         /* Tell if_ioctl and if_init that the port is going away */
 931         ADAPTER_LOCK(sc);
 932         SET_DOOMED(pi);
 933         wakeup(&sc->flags);
 934         while (IS_BUSY(sc))
 935                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 936         SET_BUSY(sc);
 937 #ifdef INVARIANTS
 938         sc->last_op = "t4detach";
 939         sc->last_op_thr = curthread;
 940 #endif
 941         ADAPTER_UNLOCK(sc);
 942
 943         if (pi->vlan_c)
 944                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
 945
 946         PORT_LOCK(pi);
 947         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 948         callout_stop(&pi->tick);
 949         PORT_UNLOCK(pi);
 950         callout_drain(&pi->tick);
 951
 952         /* Let detach proceed even if these fail. */
 953         cxgbe_uninit_synchronized(pi);
 954         port_full_uninit(pi);
 955
 956         ifmedia_removeall(&pi->media);
 957         ether_ifdetach(pi->ifp);
 958         if_free(pi->ifp);
 959
 960         ADAPTER_LOCK(sc);
 961         CLR_BUSY(sc);
 962         wakeup(&sc->flags);
 963         ADAPTER_UNLOCK(sc);
 964
 965         return (0);
 966 }
 967
 968 static void
 969 cxgbe_init(void *arg)
 970 {
 971         struct port_info *pi = arg;
 972         struct adapter *sc = pi->adapter;
 973
 974         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
 975                 return;
 976         cxgbe_init_synchronized(pi);
 977         end_synchronized_op(sc, 0);
 978 }
 979
 980 static int
 981 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 982 {
 983         int rc = 0, mtu, flags;
 984         struct port_info *pi = ifp->if_softc;
 985         struct adapter *sc = pi->adapter;
 986         struct ifreq *ifr = (struct ifreq *)data;
 987         uint32_t mask;
 988
 989         switch (cmd) {
 990         case SIOCSIFMTU:
 991                 mtu = ifr->ifr_mtu;
 992                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
 993                         return (EINVAL);
 994
 995                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
 996                 if (rc)
 997                         return (rc);
 998                 ifp->if_mtu = mtu;
 999                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1000                         t4_update_fl_bufsize(ifp);
1001                         rc = update_mac_settings(pi, XGMAC_MTU);
1002                 }
1003                 end_synchronized_op(sc, 0);
1004                 break;
1005
1006         case SIOCSIFFLAGS:
1007                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1008                 if (rc)
1009                         return (rc);
1010
1011                 if (ifp->if_flags & IFF_UP) {
1012                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013                                 flags = pi->if_flags;
1014                                 if ((ifp->if_flags ^ flags) &
1015                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1016                                         rc = update_mac_settings(pi,
1017                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1018                                 }
1019                         } else
1020                                 rc = cxgbe_init_synchronized(pi);
1021                         pi->if_flags = ifp->if_flags;
1022                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1023                         rc = cxgbe_uninit_synchronized(pi);
1024                 end_synchronized_op(sc, 0);
1025                 break;
1026
1027         case SIOCADDMULTI:
1028         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1029                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1030                 if (rc)
1031                         return (rc);
1032                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1033                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1034                 end_synchronized_op(sc, LOCK_HELD);
1035                 break;
1036
1037         case SIOCSIFCAP:
1038                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1039                 if (rc)
1040                         return (rc);
1041
1042                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1043                 if (mask & IFCAP_TXCSUM) {
1044                         ifp->if_capenable ^= IFCAP_TXCSUM;
1045                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1046
1047                         if (IFCAP_TSO4 & ifp->if_capenable &&
1048                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1049                                 ifp->if_capenable &= ~IFCAP_TSO4;
1050                                 if_printf(ifp,
1051                                     "tso4 disabled due to -txcsum.\n");
1052                         }
1053                 }
1054                 if (mask & IFCAP_TXCSUM_IPV6) {
1055                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1056                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1057
1058                         if (IFCAP_TSO6 & ifp->if_capenable &&
1059                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1060                                 ifp->if_capenable &= ~IFCAP_TSO6;
1061                                 if_printf(ifp,
1062                                     "tso6 disabled due to -txcsum6.\n");
1063                         }
1064                 }
1065                 if (mask & IFCAP_RXCSUM)
1066                         ifp->if_capenable ^= IFCAP_RXCSUM;
1067                 if (mask & IFCAP_RXCSUM_IPV6)
1068                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1069
1070                 /*
1071                  * Note that we leave CSUM_TSO alone (it is always set).  The
1072                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1073                  * sending a TSO request our way, so it's sufficient to toggle
1074                  * IFCAP_TSOx only.
1075                  */
1076                 if (mask & IFCAP_TSO4) {
1077                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1078                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1079                                 if_printf(ifp, "enable txcsum first.\n");
1080                                 rc = EAGAIN;
1081                                 goto fail;
1082                         }
1083                         ifp->if_capenable ^= IFCAP_TSO4;
1084                 }
1085                 if (mask & IFCAP_TSO6) {
1086                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1087                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1088                                 if_printf(ifp, "enable txcsum6 first.\n");
1089                                 rc = EAGAIN;
1090                                 goto fail;
1091                         }
1092                         ifp->if_capenable ^= IFCAP_TSO6;
1093                 }
1094                 if (mask & IFCAP_LRO) {
1095 #if defined(INET) || defined(INET6)
1096                         int i;
1097                         struct sge_rxq *rxq;
1098
1099                         ifp->if_capenable ^= IFCAP_LRO;
1100                         for_each_rxq(pi, i, rxq) {
1101                                 if (ifp->if_capenable & IFCAP_LRO)
1102                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1103                                 else
1104                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1105                         }
1106 #endif
1107                 }
1108 #ifdef TCP_OFFLOAD
1109                 if (mask & IFCAP_TOE) {
1110                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1111
1112                         rc = toe_capability(pi, enable);
1113                         if (rc != 0)
1114                                 goto fail;
1115
1116                         ifp->if_capenable ^= mask;
1117                 }
1118 #endif
1119                 if (mask & IFCAP_VLAN_HWTAGGING) {
1120                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1121                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1122                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1123                 }
1124                 if (mask & IFCAP_VLAN_MTU) {
1125                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1126
1127                         /* Need to find out how to disable auto-mtu-inflation */
1128                 }
1129                 if (mask & IFCAP_VLAN_HWTSO)
1130                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1131                 if (mask & IFCAP_VLAN_HWCSUM)
1132                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1133
1134 #ifdef VLAN_CAPABILITIES
1135                 VLAN_CAPABILITIES(ifp);
1136 #endif
1137 fail:
1138                 end_synchronized_op(sc, 0);
1139                 break;
1140
1141         case SIOCSIFMEDIA:
1142         case SIOCGIFMEDIA:
1143                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1144                 break;
1145
1146         default:
1147                 rc = ether_ioctl(ifp, cmd, data);
1148         }
1149
1150         return (rc);
1151 }
1152
1153 static int
1154 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1155 {
1156         struct port_info *pi = ifp->if_softc;
1157         struct adapter *sc = pi->adapter;
1158         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1159         struct buf_ring *br;
1160         int rc;
1161
1162         M_ASSERTPKTHDR(m);
1163
1164         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1165                 m_freem(m);
1166                 return (ENETDOWN);
1167         }
1168
1169         if (m->m_flags & M_FLOWID)
1170                 txq += (m->m_pkthdr.flowid % pi->ntxq);
1171         br = txq->br;
1172
1173         if (TXQ_TRYLOCK(txq) == 0) {
1174                 struct sge_eq *eq = &txq->eq;
1175
1176                 /*
1177                  * It is possible that t4_eth_tx finishes up and releases the
1178                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1179                  * need to make sure that this mbuf doesn't just sit there in
1180                  * the drbr.
1181                  */
1182
1183                 rc = drbr_enqueue(ifp, br, m);
1184                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1185                     !(eq->flags & EQ_DOOMED))
1186                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1187                 return (rc);
1188         }
1189
1190         /*
1191          * txq->m is the mbuf that is held up due to a temporary shortage of
1192          * resources and it should be put on the wire first.  Then what's in
1193          * drbr and finally the mbuf that was just passed in to us.
1194          *
1195          * Return code should indicate the fate of the mbuf that was passed in
1196          * this time.
1197          */
1198
1199         TXQ_LOCK_ASSERT_OWNED(txq);
1200         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1201
1202                 /* Queued for transmission. */
1203
1204                 rc = drbr_enqueue(ifp, br, m);
1205                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1206                 (void) t4_eth_tx(ifp, txq, m);
1207                 TXQ_UNLOCK(txq);
1208                 return (rc);
1209         }
1210
1211         /* Direct transmission. */
1212         rc = t4_eth_tx(ifp, txq, m);
1213         if (rc != 0 && txq->m)
1214                 rc = 0; /* held, will be transmitted soon (hopefully) */
1215
1216         TXQ_UNLOCK(txq);
1217         return (rc);
1218 }
1219
1220 static void
1221 cxgbe_qflush(struct ifnet *ifp)
1222 {
1223         struct port_info *pi = ifp->if_softc;
1224         struct sge_txq *txq;
1225         int i;
1226         struct mbuf *m;
1227
1228         /* queues do not exist if !PORT_INIT_DONE. */
1229         if (pi->flags & PORT_INIT_DONE) {
1230                 for_each_txq(pi, i, txq) {
1231                         TXQ_LOCK(txq);
1232                         m_freem(txq->m);
1233                         txq->m = NULL;
1234                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1235                                 m_freem(m);
1236                         TXQ_UNLOCK(txq);
1237                 }
1238         }
1239         if_qflush(ifp);
1240 }
1241
1242 static int
1243 cxgbe_media_change(struct ifnet *ifp)
1244 {
1245         struct port_info *pi = ifp->if_softc;
1246
1247         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1248
1249         return (EOPNOTSUPP);
1250 }
1251
1252 static void
1253 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1254 {
1255         struct port_info *pi = ifp->if_softc;
1256         struct ifmedia_entry *cur = pi->media.ifm_cur;
1257         int speed = pi->link_cfg.speed;
1258         int data = (pi->port_type << 8) | pi->mod_type;
1259
1260         if (cur->ifm_data != data) {
1261                 build_medialist(pi);
1262                 cur = pi->media.ifm_cur;
1263         }
1264
1265         ifmr->ifm_status = IFM_AVALID;
1266         if (!pi->link_cfg.link_ok)
1267                 return;
1268
1269         ifmr->ifm_status |= IFM_ACTIVE;
1270
1271         /* active and current will differ iff current media is autoselect. */
1272         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1273                 return;
1274
1275         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1276         if (speed == SPEED_10000)
1277                 ifmr->ifm_active |= IFM_10G_T;
1278         else if (speed == SPEED_1000)
1279                 ifmr->ifm_active |= IFM_1000_T;
1280         else if (speed == SPEED_100)
1281                 ifmr->ifm_active |= IFM_100_TX;
1282         else if (speed == SPEED_10)
1283                 ifmr->ifm_active |= IFM_10_T;
1284         else
1285                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1286                             speed));
1287 }
1288
1289 void
1290 t4_fatal_err(struct adapter *sc)
1291 {
1292         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1293         t4_intr_disable(sc);
1294         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1295             device_get_nameunit(sc->dev));
1296 }
1297
1298 static int
1299 map_bars(struct adapter *sc)
1300 {
1301         sc->regs_rid = PCIR_BAR(0);
1302         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1303             &sc->regs_rid, RF_ACTIVE);
1304         if (sc->regs_res == NULL) {
1305                 device_printf(sc->dev, "cannot map registers.\n");
1306                 return (ENXIO);
1307         }
1308         sc->bt = rman_get_bustag(sc->regs_res);
1309         sc->bh = rman_get_bushandle(sc->regs_res);
1310         sc->mmio_len = rman_get_size(sc->regs_res);
1311
1312         sc->msix_rid = PCIR_BAR(4);
1313         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1314             &sc->msix_rid, RF_ACTIVE);
1315         if (sc->msix_res == NULL) {
1316                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1317                 return (ENXIO);
1318         }
1319
1320         return (0);
1321 }
1322
1323 static void
1324 setup_memwin(struct adapter *sc)
1325 {
1326         uint32_t bar0;
1327
1328         /*
1329          * Read low 32b of bar0 indirectly via the hardware backdoor mechanism.
1330          * Works from within PCI passthrough environments too, where
1331          * rman_get_start() can return a different value.  We need to program
1332          * the memory window decoders with the actual addresses that will be
1333          * coming across the PCIe link.
1334          */
1335         bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1336         bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1337
1338         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1339                      (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1340                      V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1341
1342         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1343                      (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1344                      V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1345
1346         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1347                      (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1348                      V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1349
1350         /* flush */
1351         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1352 }
1353
1354 static int
1355 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1356     struct intrs_and_queues *iaq)
1357 {
1358         int rc, itype, navail, nrxq10g, nrxq1g, n;
1359         int nofldrxq10g = 0, nofldrxq1g = 0;
1360
1361         bzero(iaq, sizeof(*iaq));
1362
1363         iaq->ntxq10g = t4_ntxq10g;
1364         iaq->ntxq1g = t4_ntxq1g;
1365         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1366         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1367 #ifdef TCP_OFFLOAD
1368         if (is_offload(sc)) {
1369                 iaq->nofldtxq10g = t4_nofldtxq10g;
1370                 iaq->nofldtxq1g = t4_nofldtxq1g;
1371                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1372                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1373         }
1374 #endif
1375
1376         for (itype = INTR_MSIX; itype; itype >>= 1) {
1377
1378                 if ((itype & t4_intr_types) == 0)
1379                         continue;       /* not allowed */
1380
1381                 if (itype == INTR_MSIX)
1382                         navail = pci_msix_count(sc->dev);
1383                 else if (itype == INTR_MSI)
1384                         navail = pci_msi_count(sc->dev);
1385                 else
1386                         navail = 1;
1387 restart:
1388                 if (navail == 0)
1389                         continue;
1390
1391                 iaq->intr_type = itype;
1392                 iaq->intr_flags = 0;
1393
1394                 /*
1395                  * Best option: an interrupt vector for errors, one for the
1396                  * firmware event queue, and one each for each rxq (NIC as well
1397                  * as offload).
1398                  */
1399                 iaq->nirq = T4_EXTRA_INTR;
1400                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1401                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1402                 if (iaq->nirq <= navail &&
1403                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1404                         iaq->intr_flags |= INTR_DIRECT;
1405                         goto allocate;
1406                 }
1407
1408                 /*
1409                  * Second best option: an interrupt vector for errors, one for
1410                  * the firmware event queue, and one each for either NIC or
1411                  * offload rxq's.
1412                  */
1413                 iaq->nirq = T4_EXTRA_INTR;
1414                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1415                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1416                 if (iaq->nirq <= navail &&
1417                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1418                         goto allocate;
1419
1420                 /*
1421                  * Next best option: an interrupt vector for errors, one for the
1422                  * firmware event queue, and at least one per port.  At this
1423                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1424                  * what's available to us.
1425                  */
1426                 iaq->nirq = T4_EXTRA_INTR;
1427                 iaq->nirq += n10g + n1g;
1428                 if (iaq->nirq <= navail) {
1429                         int leftover = navail - iaq->nirq;
1430
1431                         if (n10g > 0) {
1432                                 int target = max(nrxq10g, nofldrxq10g);
1433
1434                                 n = 1;
1435                                 while (n < target && leftover >= n10g) {
1436                                         leftover -= n10g;
1437                                         iaq->nirq += n10g;
1438                                         n++;
1439                                 }
1440                                 iaq->nrxq10g = min(n, nrxq10g);
1441 #ifdef TCP_OFFLOAD
1442                                 if (is_offload(sc))
1443                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1444 #endif
1445                         }
1446
1447                         if (n1g > 0) {
1448                                 int target = max(nrxq1g, nofldrxq1g);
1449
1450                                 n = 1;
1451                                 while (n < target && leftover >= n1g) {
1452                                         leftover -= n1g;
1453                                         iaq->nirq += n1g;
1454                                         n++;
1455                                 }
1456                                 iaq->nrxq1g = min(n, nrxq1g);
1457 #ifdef TCP_OFFLOAD
1458                                 if (is_offload(sc))
1459                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1460 #endif
1461                         }
1462
1463                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1464                                 goto allocate;
1465                 }
1466
1467                 /*
1468                  * Least desirable option: one interrupt vector for everything.
1469                  */
1470                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1471 #ifdef TCP_OFFLOAD
1472                 if (is_offload(sc))
1473                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1474 #endif
1475
1476 allocate:
1477                 navail = iaq->nirq;
1478                 rc = 0;
1479                 if (itype == INTR_MSIX)
1480                         rc = pci_alloc_msix(sc->dev, &navail);
1481                 else if (itype == INTR_MSI)
1482                         rc = pci_alloc_msi(sc->dev, &navail);
1483
1484                 if (rc == 0) {
1485                         if (navail == iaq->nirq)
1486                                 return (0);
1487
1488                         /*
1489                          * Didn't get the number requested.  Use whatever number
1490                          * the kernel is willing to allocate (it's in navail).
1491                          */
1492                         device_printf(sc->dev, "fewer vectors than requested, "
1493                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1494                             itype, iaq->nirq, navail);
1495                         pci_release_msi(sc->dev);
1496                         goto restart;
1497                 }
1498
1499                 device_printf(sc->dev,
1500                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1501                     itype, rc, iaq->nirq, navail);
1502         }
1503
1504         device_printf(sc->dev,
1505             "failed to find a usable interrupt type.  "
1506             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1507             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1508
1509         return (ENXIO);
1510 }
1511
1512 /*
1513  * Is the given firmware compatible with the one the driver was compiled with?
1514  */
1515 static int
1516 fw_compatible(const struct fw_hdr *hdr)
1517 {
1518
1519         if (hdr->fw_ver == htonl(FW_VERSION))
1520                 return (1);
1521
1522         /*
1523          * XXX: Is this too conservative?  Perhaps I should limit this to the
1524          * features that are supported in the driver.
1525          */
1526         if (hdr->intfver_nic == FW_HDR_INTFVER_NIC &&
1527             hdr->intfver_vnic == FW_HDR_INTFVER_VNIC &&
1528             hdr->intfver_ofld == FW_HDR_INTFVER_OFLD &&
1529             hdr->intfver_ri == FW_HDR_INTFVER_RI &&
1530             hdr->intfver_iscsipdu == FW_HDR_INTFVER_ISCSIPDU &&
1531             hdr->intfver_iscsi == FW_HDR_INTFVER_ISCSI &&
1532             hdr->intfver_fcoepdu == FW_HDR_INTFVER_FCOEPDU &&
1533             hdr->intfver_fcoe == FW_HDR_INTFVER_FCOEPDU)
1534                 return (1);
1535
1536         return (0);
1537 }
1538
1539 /*
1540  * Install a compatible firmware (if required), establish contact with it (by
1541  * saying hello), and reset the device.  If we end up as the master driver,
1542  * partition adapter resources by providing a configuration file to the
1543  * firmware.
1544  */
1545 static int
1546 prep_firmware(struct adapter *sc)
1547 {
1548         const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1549         int rc, card_fw_usable, kld_fw_usable;
1550         enum dev_state state;
1551         struct fw_hdr *card_fw;
1552         const struct fw_hdr *kld_fw;
1553
1554         default_cfg = firmware_get(T4_CFGNAME);
1555
1556         /* Read the header of the firmware on the card */
1557         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
1558         rc = -t4_read_flash(sc, FLASH_FW_START,
1559             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
1560         if (rc == 0)
1561                 card_fw_usable = fw_compatible((const void*)card_fw);
1562         else {
1563                 device_printf(sc->dev,
1564                     "Unable to read card's firmware header: %d\n", rc);
1565                 card_fw_usable = 0;
1566         }
1567
1568         /* This is the firmware in the KLD */
1569         fw = firmware_get(T4_FWNAME);
1570         if (fw != NULL) {
1571                 kld_fw = (const void *)fw->data;
1572                 kld_fw_usable = fw_compatible(kld_fw);
1573         } else {
1574                 kld_fw = NULL;
1575                 kld_fw_usable = 0;
1576         }
1577
1578         /*
1579          * Short circuit for the common case: the firmware on the card is an
1580          * exact match and the KLD is an exact match too, or it's
1581          * absent/incompatible, or we're prohibited from using it.  Note that
1582          * t4_fw_install = 2 is ignored here -- use cxgbetool loadfw if you want
1583          * to reinstall the same firmware as the one on the card.
1584          */
1585         if (card_fw_usable && card_fw->fw_ver == htonl(FW_VERSION) &&
1586             (!kld_fw_usable || kld_fw->fw_ver == htonl(FW_VERSION) ||
1587             t4_fw_install == 0))
1588                 goto hello;
1589
1590         if (kld_fw_usable && (!card_fw_usable ||
1591             ntohl(kld_fw->fw_ver) > ntohl(card_fw->fw_ver) ||
1592             (t4_fw_install == 2 && kld_fw->fw_ver != card_fw->fw_ver))) {
1593                 uint32_t v = ntohl(kld_fw->fw_ver);
1594
1595                 device_printf(sc->dev,
1596                     "installing firmware %d.%d.%d.%d on card.\n",
1597                     G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1598                     G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1599
1600                 rc = -t4_load_fw(sc, fw->data, fw->datasize);
1601                 if (rc != 0) {
1602                         device_printf(sc->dev,
1603                             "failed to install firmware: %d\n", rc);
1604                         goto done;
1605                 }
1606
1607                 /* Installed successfully, update the cached header too. */
1608                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
1609                 card_fw_usable = 1;
1610         }
1611
1612         if (!card_fw_usable) {
1613                 uint32_t c, k;
1614
1615                 c = ntohl(card_fw->fw_ver);
1616                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
1617
1618                 device_printf(sc->dev, "Cannot find a usable firmware: "
1619                     "fw_install %d, driver compiled with %d.%d.%d.%d, "
1620                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
1621                     t4_fw_install,
1622                     G_FW_HDR_FW_VER_MAJOR(FW_VERSION),
1623                     G_FW_HDR_FW_VER_MINOR(FW_VERSION),
1624                     G_FW_HDR_FW_VER_MICRO(FW_VERSION),
1625                     G_FW_HDR_FW_VER_BUILD(FW_VERSION),
1626                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
1627                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
1628                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
1629                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
1630                 goto done;
1631         }
1632
1633 hello:
1634         /* We're using whatever's on the card and it's known to be good. */
1635         sc->params.fw_vers = ntohl(card_fw->fw_ver);
1636         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1637             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1638             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1639             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1640             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1641
1642         /* Contact firmware.  */
1643         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1644         if (rc < 0) {
1645                 rc = -rc;
1646                 device_printf(sc->dev,
1647                     "failed to connect to the firmware: %d.\n", rc);
1648                 goto done;
1649         }
1650         if (rc == sc->mbox)
1651                 sc->flags |= MASTER_PF;
1652
1653         /* Reset device */
1654         rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1655         if (rc != 0) {
1656                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1657                 if (rc != ETIMEDOUT && rc != EIO)
1658                         t4_fw_bye(sc, sc->mbox);
1659                 goto done;
1660         }
1661
1662         /* Partition adapter resources as specified in the config file. */
1663         if (sc->flags & MASTER_PF) {
1664                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s",
1665                     pci_get_device(sc->dev) == 0x440a ? "uwire" : t4_cfg_file);
1666                 if (strncmp(sc->cfg_file, "default", sizeof(sc->cfg_file))) {
1667                         char s[32];
1668
1669                         snprintf(s, sizeof(s), "t4fw_cfg_%s", sc->cfg_file);
1670                         cfg = firmware_get(s);
1671                         if (cfg == NULL) {
1672                                 device_printf(sc->dev,
1673                                     "unable to locate %s module, "
1674                                     "will use default config file.\n", s);
1675                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
1676                                     "%s", "default");
1677                         }
1678                 }
1679
1680                 rc = partition_resources(sc, cfg ? cfg : default_cfg);
1681                 if (rc != 0)
1682                         goto done;      /* error message displayed already */
1683         } else {
1684                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", "notme");
1685                 sc->cfcsum = (u_int)-1;
1686         }
1687
1688         sc->flags |= FW_OK;
1689
1690 done:
1691         free(card_fw, M_CXGBE);
1692         if (fw != NULL)
1693                 firmware_put(fw, FIRMWARE_UNLOAD);
1694         if (cfg != NULL)
1695                 firmware_put(cfg, FIRMWARE_UNLOAD);
1696         if (default_cfg != NULL)
1697                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
1698
1699         return (rc);
1700 }
1701
1702 #define FW_PARAM_DEV(param) \
1703         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1704          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1705 #define FW_PARAM_PFVF(param) \
1706         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1707          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1708
1709 /*
1710  * Upload configuration file to card's memory.
1711  */
1712 static int
1713 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1714     uint32_t *ma)
1715 {
1716         int rc, i;
1717         uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1718         const uint32_t *b;
1719
1720         /* Figure out where the firmware wants us to upload it. */
1721         param = FW_PARAM_DEV(CF);
1722         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1723         if (rc != 0) {
1724                 /* Firmwares without config file support will fail this way */
1725                 device_printf(sc->dev,
1726                     "failed to query config file location: %d.\n", rc);
1727                 return (rc);
1728         }
1729         *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1730         *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1731
1732         if (maddr & 3) {
1733                 device_printf(sc->dev,
1734                     "cannot upload config file (type %u, addr %x).\n",
1735                     mtype, maddr);
1736                 return (EFAULT);
1737         }
1738
1739         /* Translate mtype/maddr to an address suitable for the PCIe window */
1740         val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1741         val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1742         switch (mtype) {
1743         case FW_MEMTYPE_CF_EDC0:
1744                 if (!(val & F_EDRAM0_ENABLE))
1745                         goto err;
1746                 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1747                 maddr += G_EDRAM0_BASE(bar) << 20;
1748                 break;
1749
1750         case FW_MEMTYPE_CF_EDC1:
1751                 if (!(val & F_EDRAM1_ENABLE))
1752                         goto err;
1753                 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1754                 maddr += G_EDRAM1_BASE(bar) << 20;
1755                 break;
1756
1757         case FW_MEMTYPE_CF_EXTMEM:
1758                 if (!(val & F_EXT_MEM_ENABLE))
1759                         goto err;
1760                 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1761                 maddr += G_EXT_MEM_BASE(bar) << 20;
1762                 break;
1763
1764         default:
1765 err:
1766                 device_printf(sc->dev,
1767                     "cannot upload config file (type %u, enabled %u).\n",
1768                     mtype, val);
1769                 return (EFAULT);
1770         }
1771
1772         /*
1773          * Position the PCIe window (we use memwin2) to the 16B aligned area
1774          * just at/before the upload location.
1775          */
1776         win = maddr & ~0xf;
1777         off = maddr - win;  /* offset from the start of the window. */
1778         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1779         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1780
1781         remaining = fw->datasize;
1782         if (remaining > FLASH_CFG_MAX_SIZE ||
1783             remaining > MEMWIN2_APERTURE - off) {
1784                 device_printf(sc->dev, "cannot upload config file all at once "
1785                     "(size %u, max %u, room %u).\n",
1786                     remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1787                 return (EFBIG);
1788         }
1789
1790         /*
1791          * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1792          * stuffing/comments at the end of the config file so it's ok to simply
1793          * throw away the last remaining bytes when the config file is not an
1794          * exact multiple of 4.
1795          */
1796         b = fw->data;
1797         for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1798                 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1799
1800         return (rc);
1801 }
1802
1803 /*
1804  * Partition chip resources for use between various PFs, VFs, etc.  This is done
1805  * by uploading the firmware configuration file to the adapter and instructing
1806  * the firmware to process it.
1807  */
1808 static int
1809 partition_resources(struct adapter *sc, const struct firmware *cfg)
1810 {
1811         int rc;
1812         struct fw_caps_config_cmd caps;
1813         uint32_t mtype, maddr, finicsum, cfcsum;
1814
1815         rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1816         if (rc != 0) {
1817                 mtype = FW_MEMTYPE_CF_FLASH;
1818                 maddr = t4_flash_cfg_addr(sc);
1819         }
1820
1821         bzero(&caps, sizeof(caps));
1822         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1823             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1824         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1825             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1826             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1827         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1828         if (rc != 0) {
1829                 device_printf(sc->dev,
1830                     "failed to pre-process config file: %d.\n", rc);
1831                 return (rc);
1832         }
1833
1834         finicsum = be32toh(caps.finicsum);
1835         cfcsum = be32toh(caps.cfcsum);
1836         if (finicsum != cfcsum) {
1837                 device_printf(sc->dev,
1838                     "WARNING: config file checksum mismatch: %08x %08x\n",
1839                     finicsum, cfcsum);
1840         }
1841         sc->cfcsum = cfcsum;
1842
1843 #define LIMIT_CAPS(x) do { \
1844         caps.x &= htobe16(t4_##x##_allowed); \
1845         sc->x = htobe16(caps.x); \
1846 } while (0)
1847
1848         /*
1849          * Let the firmware know what features will (not) be used so it can tune
1850          * things accordingly.
1851          */
1852         LIMIT_CAPS(linkcaps);
1853         LIMIT_CAPS(niccaps);
1854         LIMIT_CAPS(toecaps);
1855         LIMIT_CAPS(rdmacaps);
1856         LIMIT_CAPS(iscsicaps);
1857         LIMIT_CAPS(fcoecaps);
1858 #undef LIMIT_CAPS
1859
1860         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1861             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1862         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1863         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1864         if (rc != 0) {
1865                 device_printf(sc->dev,
1866                     "failed to process config file: %d.\n", rc);
1867                 return (rc);
1868         }
1869
1870         return (0);
1871 }
1872
1873 /*
1874  * Retrieve parameters that are needed (or nice to have) prior to calling
1875  * t4_sge_init and t4_fw_initialize.
1876  */
1877 static int
1878 get_params__pre_init(struct adapter *sc)
1879 {
1880         int rc;
1881         uint32_t param[2], val[2];
1882         struct fw_devlog_cmd cmd;
1883         struct devlog_params *dlog = &sc->params.devlog;
1884
1885         param[0] = FW_PARAM_DEV(PORTVEC);
1886         param[1] = FW_PARAM_DEV(CCLK);
1887         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1888         if (rc != 0) {
1889                 device_printf(sc->dev,
1890                     "failed to query parameters (pre_init): %d.\n", rc);
1891                 return (rc);
1892         }
1893
1894         sc->params.portvec = val[0];
1895         sc->params.nports = bitcount32(val[0]);
1896         sc->params.vpd.cclk = val[1];
1897
1898         /* Read device log parameters. */
1899         bzero(&cmd, sizeof(cmd));
1900         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1901             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1902         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1903         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1904         if (rc != 0) {
1905                 device_printf(sc->dev,
1906                     "failed to get devlog parameters: %d.\n", rc);
1907                 bzero(dlog, sizeof (*dlog));
1908                 rc = 0; /* devlog isn't critical for device operation */
1909         } else {
1910                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1911                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1912                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1913                 dlog->size = be32toh(cmd.memsize_devlog);
1914         }
1915
1916         return (rc);
1917 }
1918
1919 /*
1920  * Retrieve various parameters that are of interest to the driver.  The device
1921  * has been initialized by the firmware at this point.
1922  */
1923 static int
1924 get_params__post_init(struct adapter *sc)
1925 {
1926         int rc;
1927         uint32_t param[7], val[7];
1928         struct fw_caps_config_cmd caps;
1929
1930         param[0] = FW_PARAM_PFVF(IQFLINT_START);
1931         param[1] = FW_PARAM_PFVF(EQ_START);
1932         param[2] = FW_PARAM_PFVF(FILTER_START);
1933         param[3] = FW_PARAM_PFVF(FILTER_END);
1934         param[4] = FW_PARAM_PFVF(L2T_START);
1935         param[5] = FW_PARAM_PFVF(L2T_END);
1936         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1937         if (rc != 0) {
1938                 device_printf(sc->dev,
1939                     "failed to query parameters (post_init): %d.\n", rc);
1940                 return (rc);
1941         }
1942
1943         sc->sge.iq_start = val[0];
1944         sc->sge.eq_start = val[1];
1945         sc->tids.ftid_base = val[2];
1946         sc->tids.nftids = val[3] - val[2] + 1;
1947         sc->vres.l2t.start = val[4];
1948         sc->vres.l2t.size = val[5] - val[4] + 1;
1949         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
1950             ("%s: L2 table size (%u) larger than expected (%u)",
1951             __func__, sc->vres.l2t.size, L2T_SIZE));
1952
1953         /* get capabilites */
1954         bzero(&caps, sizeof(caps));
1955         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1956             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1957         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1958         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1959         if (rc != 0) {
1960                 device_printf(sc->dev,
1961                     "failed to get card capabilities: %d.\n", rc);
1962                 return (rc);
1963         }
1964
1965         if (caps.toecaps) {
1966                 /* query offload-related parameters */
1967                 param[0] = FW_PARAM_DEV(NTID);
1968                 param[1] = FW_PARAM_PFVF(SERVER_START);
1969                 param[2] = FW_PARAM_PFVF(SERVER_END);
1970                 param[3] = FW_PARAM_PFVF(TDDP_START);
1971                 param[4] = FW_PARAM_PFVF(TDDP_END);
1972                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1973                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1974                 if (rc != 0) {
1975                         device_printf(sc->dev,
1976                             "failed to query TOE parameters: %d.\n", rc);
1977                         return (rc);
1978                 }
1979                 sc->tids.ntids = val[0];
1980                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1981                 sc->tids.stid_base = val[1];
1982                 sc->tids.nstids = val[2] - val[1] + 1;
1983                 sc->vres.ddp.start = val[3];
1984                 sc->vres.ddp.size = val[4] - val[3] + 1;
1985                 sc->params.ofldq_wr_cred = val[5];
1986                 sc->params.offload = 1;
1987         }
1988         if (caps.rdmacaps) {
1989                 param[0] = FW_PARAM_PFVF(STAG_START);
1990                 param[1] = FW_PARAM_PFVF(STAG_END);
1991                 param[2] = FW_PARAM_PFVF(RQ_START);
1992                 param[3] = FW_PARAM_PFVF(RQ_END);
1993                 param[4] = FW_PARAM_PFVF(PBL_START);
1994                 param[5] = FW_PARAM_PFVF(PBL_END);
1995                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1996                 if (rc != 0) {
1997                         device_printf(sc->dev,
1998                             "failed to query RDMA parameters(1): %d.\n", rc);
1999                         return (rc);
2000                 }
2001                 sc->vres.stag.start = val[0];
2002                 sc->vres.stag.size = val[1] - val[0] + 1;
2003                 sc->vres.rq.start = val[2];
2004                 sc->vres.rq.size = val[3] - val[2] + 1;
2005                 sc->vres.pbl.start = val[4];
2006                 sc->vres.pbl.size = val[5] - val[4] + 1;
2007
2008                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2009                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2010                 param[2] = FW_PARAM_PFVF(CQ_START);
2011                 param[3] = FW_PARAM_PFVF(CQ_END);
2012                 param[4] = FW_PARAM_PFVF(OCQ_START);
2013                 param[5] = FW_PARAM_PFVF(OCQ_END);
2014                 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
2015                 if (rc != 0) {
2016                         device_printf(sc->dev,
2017                             "failed to query RDMA parameters(2): %d.\n", rc);
2018                         return (rc);
2019                 }
2020                 sc->vres.qp.start = val[0];
2021                 sc->vres.qp.size = val[1] - val[0] + 1;
2022                 sc->vres.cq.start = val[2];
2023                 sc->vres.cq.size = val[3] - val[2] + 1;
2024                 sc->vres.ocq.start = val[4];
2025                 sc->vres.ocq.size = val[5] - val[4] + 1;
2026         }
2027         if (caps.iscsicaps) {
2028                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2029                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2030                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2031                 if (rc != 0) {
2032                         device_printf(sc->dev,
2033                             "failed to query iSCSI parameters: %d.\n", rc);
2034                         return (rc);
2035                 }
2036                 sc->vres.iscsi.start = val[0];
2037                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2038         }
2039
2040         /* These are finalized by FW initialization, load their values now */
2041         val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
2042         sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
2043         sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
2044         t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
2045
2046         return (rc);
2047 }
2048
2049 static int
2050 set_params__post_init(struct adapter *sc)
2051 {
2052         uint32_t param, val;
2053         int rc;
2054
2055         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2056         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2057         if (rc == 0) {
2058                 /* ask for encapsulated CPLs */
2059                 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2060                 val = 1;
2061                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2062                 if (rc != 0) {
2063                         device_printf(sc->dev,
2064                             "failed to set parameter (post_init): %d.\n", rc);
2065                         return (rc);
2066                 }
2067         } else if (rc != FW_EINVAL) {
2068                 device_printf(sc->dev,
2069                     "failed to check for encapsulated CPLs: %d.\n", rc);
2070         } else
2071                 rc = 0; /* the firmware doesn't support the param, no worries */
2072
2073         return (rc);
2074 }
2075
2076 #undef FW_PARAM_PFVF
2077 #undef FW_PARAM_DEV
2078
2079 static void
2080 t4_set_desc(struct adapter *sc)
2081 {
2082         char buf[128];
2083         struct adapter_params *p = &sc->params;
2084
2085         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
2086             p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
2087
2088         device_set_desc_copy(sc->dev, buf);
2089 }
2090
2091 static void
2092 build_medialist(struct port_info *pi)
2093 {
2094         struct ifmedia *media = &pi->media;
2095         int data, m;
2096
2097         PORT_LOCK(pi);
2098
2099         ifmedia_removeall(media);
2100
2101         m = IFM_ETHER | IFM_FDX;
2102         data = (pi->port_type << 8) | pi->mod_type;
2103
2104         switch(pi->port_type) {
2105         case FW_PORT_TYPE_BT_XFI:
2106                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2107                 break;
2108
2109         case FW_PORT_TYPE_BT_XAUI:
2110                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2111                 /* fall through */
2112
2113         case FW_PORT_TYPE_BT_SGMII:
2114                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2115                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2116                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2117                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2118                 break;
2119
2120         case FW_PORT_TYPE_CX4:
2121                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2122                 ifmedia_set(media, m | IFM_10G_CX4);
2123                 break;
2124
2125         case FW_PORT_TYPE_SFP:
2126         case FW_PORT_TYPE_FIBER_XFI:
2127         case FW_PORT_TYPE_FIBER_XAUI:
2128                 switch (pi->mod_type) {
2129
2130                 case FW_PORT_MOD_TYPE_LR:
2131                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2132                         ifmedia_set(media, m | IFM_10G_LR);
2133                         break;
2134
2135                 case FW_PORT_MOD_TYPE_SR:
2136                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2137                         ifmedia_set(media, m | IFM_10G_SR);
2138                         break;
2139
2140                 case FW_PORT_MOD_TYPE_LRM:
2141                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2142                         ifmedia_set(media, m | IFM_10G_LRM);
2143                         break;
2144
2145                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2146                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2147                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2148                         ifmedia_set(media, m | IFM_10G_TWINAX);
2149                         break;
2150
2151                 case FW_PORT_MOD_TYPE_NONE:
2152                         m &= ~IFM_FDX;
2153                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2154                         ifmedia_set(media, m | IFM_NONE);
2155                         break;
2156
2157                 case FW_PORT_MOD_TYPE_NA:
2158                 case FW_PORT_MOD_TYPE_ER:
2159                 default:
2160                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2161                         ifmedia_set(media, m | IFM_UNKNOWN);
2162                         break;
2163                 }
2164                 break;
2165
2166         case FW_PORT_TYPE_KX4:
2167         case FW_PORT_TYPE_KX:
2168         case FW_PORT_TYPE_KR:
2169         default:
2170                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2171                 ifmedia_set(media, m | IFM_UNKNOWN);
2172                 break;
2173         }
2174
2175         PORT_UNLOCK(pi);
2176 }
2177
2178 #define FW_MAC_EXACT_CHUNK      7
2179
2180 /*
2181  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2182  * indicates which parameters should be programmed (the rest are left alone).
2183  */
2184 static int
2185 update_mac_settings(struct port_info *pi, int flags)
2186 {
2187         int rc;
2188         struct ifnet *ifp = pi->ifp;
2189         struct adapter *sc = pi->adapter;
2190         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2191
2192         ASSERT_SYNCHRONIZED_OP(sc);
2193         KASSERT(flags, ("%s: not told what to update.", __func__));
2194
2195         if (flags & XGMAC_MTU)
2196                 mtu = ifp->if_mtu;
2197
2198         if (flags & XGMAC_PROMISC)
2199                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2200
2201         if (flags & XGMAC_ALLMULTI)
2202                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2203
2204         if (flags & XGMAC_VLANEX)
2205                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2206
2207         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2208             vlanex, false);
2209         if (rc) {
2210                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2211                 return (rc);
2212         }
2213
2214         if (flags & XGMAC_UCADDR) {
2215                 uint8_t ucaddr[ETHER_ADDR_LEN];
2216
2217                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2218                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2219                     ucaddr, true, true);
2220                 if (rc < 0) {
2221                         rc = -rc;
2222                         if_printf(ifp, "change_mac failed: %d\n", rc);
2223                         return (rc);
2224                 } else {
2225                         pi->xact_addr_filt = rc;
2226                         rc = 0;
2227                 }
2228         }
2229
2230         if (flags & XGMAC_MCADDRS) {
2231                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2232                 int del = 1;
2233                 uint64_t hash = 0;
2234                 struct ifmultiaddr *ifma;
2235                 int i = 0, j;
2236
2237                 if_maddr_rlock(ifp);
2238                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2239                         if (ifma->ifma_addr->sa_family != AF_LINK)
2240                                 continue;
2241                         mcaddr[i++] =
2242                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2243
2244                         if (i == FW_MAC_EXACT_CHUNK) {
2245                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2246                                     del, i, mcaddr, NULL, &hash, 0);
2247                                 if (rc < 0) {
2248                                         rc = -rc;
2249                                         for (j = 0; j < i; j++) {
2250                                                 if_printf(ifp,
2251                                                     "failed to add mc address"
2252                                                     " %02x:%02x:%02x:"
2253                                                     "%02x:%02x:%02x rc=%d\n",
2254                                                     mcaddr[j][0], mcaddr[j][1],
2255                                                     mcaddr[j][2], mcaddr[j][3],
2256                                                     mcaddr[j][4], mcaddr[j][5],
2257                                                     rc);
2258                                         }
2259                                         goto mcfail;
2260                                 }
2261                                 del = 0;
2262                                 i = 0;
2263                         }
2264                 }
2265                 if (i > 0) {
2266                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2267                             del, i, mcaddr, NULL, &hash, 0);
2268                         if (rc < 0) {
2269                                 rc = -rc;
2270                                 for (j = 0; j < i; j++) {
2271                                         if_printf(ifp,
2272                                             "failed to add mc address"
2273                                             " %02x:%02x:%02x:"
2274                                             "%02x:%02x:%02x rc=%d\n",
2275                                             mcaddr[j][0], mcaddr[j][1],
2276                                             mcaddr[j][2], mcaddr[j][3],
2277                                             mcaddr[j][4], mcaddr[j][5],
2278                                             rc);
2279                                 }
2280                                 goto mcfail;
2281                         }
2282                 }
2283
2284                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2285                 if (rc != 0)
2286                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2287 mcfail:
2288                 if_maddr_runlock(ifp);
2289         }
2290
2291         return (rc);
2292 }
2293
2294 int
2295 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2296     char *wmesg)
2297 {
2298         int rc, pri;
2299
2300 #ifdef WITNESS
2301         /* the caller thinks it's ok to sleep, but is it really? */
2302         if (flags & SLEEP_OK)
2303                 pause("t4slptst", 1);
2304 #endif
2305
2306         if (INTR_OK)
2307                 pri = PCATCH;
2308         else
2309                 pri = 0;
2310
2311         ADAPTER_LOCK(sc);
2312         for (;;) {
2313
2314                 if (pi && IS_DOOMED(pi)) {
2315                         rc = ENXIO;
2316                         goto done;
2317                 }
2318
2319                 if (!IS_BUSY(sc)) {
2320                         rc = 0;
2321                         break;
2322                 }
2323
2324                 if (!(flags & SLEEP_OK)) {
2325                         rc = EBUSY;
2326                         goto done;
2327                 }
2328
2329                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2330                         rc = EINTR;
2331                         goto done;
2332                 }
2333         }
2334
2335         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2336         SET_BUSY(sc);
2337 #ifdef INVARIANTS
2338         sc->last_op = wmesg;
2339         sc->last_op_thr = curthread;
2340 #endif
2341
2342 done:
2343         if (!(flags & HOLD_LOCK) || rc)
2344                 ADAPTER_UNLOCK(sc);
2345
2346         return (rc);
2347 }
2348
2349 void
2350 end_synchronized_op(struct adapter *sc, int flags)
2351 {
2352
2353         if (flags & LOCK_HELD)
2354                 ADAPTER_LOCK_ASSERT_OWNED(sc);
2355         else
2356                 ADAPTER_LOCK(sc);
2357
2358         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2359         CLR_BUSY(sc);
2360         wakeup(&sc->flags);
2361         ADAPTER_UNLOCK(sc);
2362 }
2363
2364 static int
2365 cxgbe_init_synchronized(struct port_info *pi)
2366 {
2367         struct adapter *sc = pi->adapter;
2368         struct ifnet *ifp = pi->ifp;
2369         int rc = 0;
2370
2371         ASSERT_SYNCHRONIZED_OP(sc);
2372
2373         if (isset(&sc->open_device_map, pi->port_id)) {
2374                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2375                     ("mismatch between open_device_map and if_drv_flags"));
2376                 return (0);     /* already running */
2377         }
2378
2379         if (!(sc->flags & FULL_INIT_DONE) &&
2380             ((rc = adapter_full_init(sc)) != 0))
2381                 return (rc);    /* error message displayed already */
2382
2383         if (!(pi->flags & PORT_INIT_DONE) &&
2384             ((rc = port_full_init(pi)) != 0))
2385                 return (rc); /* error message displayed already */
2386
2387         rc = update_mac_settings(pi, XGMAC_ALL);
2388         if (rc)
2389                 goto done;      /* error message displayed already */
2390
2391         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2392         if (rc != 0) {
2393                 if_printf(ifp, "start_link failed: %d\n", rc);
2394                 goto done;
2395         }
2396
2397         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2398         if (rc != 0) {
2399                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2400                 goto done;
2401         }
2402
2403         /* all ok */
2404         setbit(&sc->open_device_map, pi->port_id);
2405         PORT_LOCK(pi);
2406         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2407         PORT_UNLOCK(pi);
2408
2409         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2410 done:
2411         if (rc != 0)
2412                 cxgbe_uninit_synchronized(pi);
2413
2414         return (rc);
2415 }
2416
2417 /*
2418  * Idempotent.
2419  */
2420 static int
2421 cxgbe_uninit_synchronized(struct port_info *pi)
2422 {
2423         struct adapter *sc = pi->adapter;
2424         struct ifnet *ifp = pi->ifp;
2425         int rc;
2426
2427         ASSERT_SYNCHRONIZED_OP(sc);
2428
2429         /*
2430          * Disable the VI so that all its data in either direction is discarded
2431          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2432          * tick) intact as the TP can deliver negative advice or data that it's
2433          * holding in its RAM (for an offloaded connection) even after the VI is
2434          * disabled.
2435          */
2436         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2437         if (rc) {
2438                 if_printf(ifp, "disable_vi failed: %d\n", rc);
2439                 return (rc);
2440         }
2441
2442         clrbit(&sc->open_device_map, pi->port_id);
2443         PORT_LOCK(pi);
2444         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2445         PORT_UNLOCK(pi);
2446
2447         pi->link_cfg.link_ok = 0;
2448         pi->link_cfg.speed = 0;
2449         t4_os_link_changed(sc, pi->port_id, 0);
2450
2451         return (0);
2452 }
2453
2454 /*
2455  * It is ok for this function to fail midway and return right away.  t4_detach
2456  * will walk the entire sc->irq list and clean up whatever is valid.
2457  */
2458 static int
2459 setup_intr_handlers(struct adapter *sc)
2460 {
2461         int rc, rid, p, q;
2462         char s[8];
2463         struct irq *irq;
2464         struct port_info *pi;
2465         struct sge_rxq *rxq;
2466 #ifdef TCP_OFFLOAD
2467         struct sge_ofld_rxq *ofld_rxq;
2468 #endif
2469
2470         /*
2471          * Setup interrupts.
2472          */
2473         irq = &sc->irq[0];
2474         rid = sc->intr_type == INTR_INTX ? 0 : 1;
2475         if (sc->intr_count == 1) {
2476                 KASSERT(!(sc->flags & INTR_DIRECT),
2477                     ("%s: single interrupt && INTR_DIRECT?", __func__));
2478
2479                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
2480                 if (rc != 0)
2481                         return (rc);
2482         } else {
2483                 /* Multiple interrupts. */
2484                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2485                     ("%s: too few intr.", __func__));
2486
2487                 /* The first one is always error intr */
2488                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
2489                 if (rc != 0)
2490                         return (rc);
2491                 irq++;
2492                 rid++;
2493
2494                 /* The second one is always the firmware event queue */
2495                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
2496                     "evt");
2497                 if (rc != 0)
2498                         return (rc);
2499                 irq++;
2500                 rid++;
2501
2502                 /*
2503                  * Note that if INTR_DIRECT is not set then either the NIC rx
2504                  * queues or (exclusive or) the TOE rx queueus will be taking
2505                  * direct interrupts.
2506                  *
2507                  * There is no need to check for is_offload(sc) as nofldrxq
2508                  * will be 0 if offload is disabled.
2509                  */
2510                 for_each_port(sc, p) {
2511                         pi = sc->port[p];
2512
2513 #ifdef TCP_OFFLOAD
2514                         /*
2515                          * Skip over the NIC queues if they aren't taking direct
2516                          * interrupts.
2517                          */
2518                         if (!(sc->flags & INTR_DIRECT) &&
2519                             pi->nofldrxq > pi->nrxq)
2520                                 goto ofld_queues;
2521 #endif
2522                         rxq = &sc->sge.rxq[pi->first_rxq];
2523                         for (q = 0; q < pi->nrxq; q++, rxq++) {
2524                                 snprintf(s, sizeof(s), "%d.%d", p, q);
2525                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
2526                                     s);
2527                                 if (rc != 0)
2528                                         return (rc);
2529                                 irq++;
2530                                 rid++;
2531                         }
2532
2533 #ifdef TCP_OFFLOAD
2534                         /*
2535                          * Skip over the offload queues if they aren't taking
2536                          * direct interrupts.
2537                          */
2538                         if (!(sc->flags & INTR_DIRECT))
2539                                 continue;
2540 ofld_queues:
2541                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2542                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2543                                 snprintf(s, sizeof(s), "%d,%d", p, q);
2544                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
2545                                     ofld_rxq, s);
2546                                 if (rc != 0)
2547                                         return (rc);
2548                                 irq++;
2549                                 rid++;
2550                         }
2551 #endif
2552                 }
2553         }
2554
2555         return (0);
2556 }
2557
2558 static int
2559 adapter_full_init(struct adapter *sc)
2560 {
2561         int rc, i;
2562
2563         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2564         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2565             ("%s: FULL_INIT_DONE already", __func__));
2566
2567         /*
2568          * queues that belong to the adapter (not any particular port).
2569          */
2570         rc = t4_setup_adapter_queues(sc);
2571         if (rc != 0)
2572                 goto done;
2573
2574         for (i = 0; i < nitems(sc->tq); i++) {
2575                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2576                     taskqueue_thread_enqueue, &sc->tq[i]);
2577                 if (sc->tq[i] == NULL) {
2578                         device_printf(sc->dev,
2579                             "failed to allocate task queue %d\n", i);
2580                         rc = ENOMEM;
2581                         goto done;
2582                 }
2583                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2584                     device_get_nameunit(sc->dev), i);
2585         }
2586
2587         t4_intr_enable(sc);
2588         sc->flags |= FULL_INIT_DONE;
2589 done:
2590         if (rc != 0)
2591                 adapter_full_uninit(sc);
2592
2593         return (rc);
2594 }
2595
2596 static int
2597 adapter_full_uninit(struct adapter *sc)
2598 {
2599         int i;
2600
2601         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2602
2603         t4_teardown_adapter_queues(sc);
2604
2605         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
2606                 taskqueue_free(sc->tq[i]);
2607                 sc->tq[i] = NULL;
2608         }
2609
2610         sc->flags &= ~FULL_INIT_DONE;
2611
2612         return (0);
2613 }
2614
2615 static int
2616 port_full_init(struct port_info *pi)
2617 {
2618         struct adapter *sc = pi->adapter;
2619         struct ifnet *ifp = pi->ifp;
2620         uint16_t *rss;
2621         struct sge_rxq *rxq;
2622         int rc, i;
2623
2624         ASSERT_SYNCHRONIZED_OP(sc);
2625         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2626             ("%s: PORT_INIT_DONE already", __func__));
2627
2628         sysctl_ctx_init(&pi->ctx);
2629         pi->flags |= PORT_SYSCTL_CTX;
2630
2631         /*
2632          * Allocate tx/rx/fl queues for this port.
2633          */
2634         rc = t4_setup_port_queues(pi);
2635         if (rc != 0)
2636                 goto done;      /* error message displayed already */
2637
2638         /*
2639          * Setup RSS for this port.
2640          */
2641         rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2642             M_ZERO | M_WAITOK);
2643         for_each_rxq(pi, i, rxq) {
2644                 rss[i] = rxq->iq.abs_id;
2645         }
2646         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2647             pi->rss_size, rss, pi->nrxq);
2648         free(rss, M_CXGBE);
2649         if (rc != 0) {
2650                 if_printf(ifp, "rss_config failed: %d\n", rc);
2651                 goto done;
2652         }
2653
2654         pi->flags |= PORT_INIT_DONE;
2655 done:
2656         if (rc != 0)
2657                 port_full_uninit(pi);
2658
2659         return (rc);
2660 }
2661
2662 /*
2663  * Idempotent.
2664  */
2665 static int
2666 port_full_uninit(struct port_info *pi)
2667 {
2668         struct adapter *sc = pi->adapter;
2669         int i;
2670         struct sge_rxq *rxq;
2671         struct sge_txq *txq;
2672 #ifdef TCP_OFFLOAD
2673         struct sge_ofld_rxq *ofld_rxq;
2674         struct sge_wrq *ofld_txq;
2675 #endif
2676
2677         if (pi->flags & PORT_INIT_DONE) {
2678
2679                 /* Need to quiesce queues.  XXX: ctrl queues? */
2680
2681                 for_each_txq(pi, i, txq) {
2682                         quiesce_eq(sc, &txq->eq);
2683                 }
2684
2685 #ifdef TCP_OFFLOAD
2686                 for_each_ofld_txq(pi, i, ofld_txq) {
2687                         quiesce_eq(sc, &ofld_txq->eq);
2688                 }
2689 #endif
2690
2691                 for_each_rxq(pi, i, rxq) {
2692                         quiesce_iq(sc, &rxq->iq);
2693                         quiesce_fl(sc, &rxq->fl);
2694                 }
2695
2696 #ifdef TCP_OFFLOAD
2697                 for_each_ofld_rxq(pi, i, ofld_rxq) {
2698                         quiesce_iq(sc, &ofld_rxq->iq);
2699                         quiesce_fl(sc, &ofld_rxq->fl);
2700                 }
2701 #endif
2702         }
2703
2704         t4_teardown_port_queues(pi);
2705         pi->flags &= ~PORT_INIT_DONE;
2706
2707         return (0);
2708 }
2709
2710 static void
2711 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
2712 {
2713         EQ_LOCK(eq);
2714         eq->flags |= EQ_DOOMED;
2715
2716         /*
2717          * Wait for the response to a credit flush if one's
2718          * pending.
2719          */
2720         while (eq->flags & EQ_CRFLUSHED)
2721                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2722         EQ_UNLOCK(eq);
2723
2724         callout_drain(&eq->tx_callout); /* XXX: iffy */
2725         pause("callout", 10);           /* Still iffy */
2726
2727         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2728 }
2729
2730 static void
2731 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
2732 {
2733         (void) sc;      /* unused */
2734
2735         /* Synchronize with the interrupt handler */
2736         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2737                 pause("iqfree", 1);
2738 }
2739
2740 static void
2741 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
2742 {
2743         mtx_lock(&sc->sfl_lock);
2744         FL_LOCK(fl);
2745         fl->flags |= FL_DOOMED;
2746         FL_UNLOCK(fl);
2747         mtx_unlock(&sc->sfl_lock);
2748
2749         callout_drain(&sc->sfl_callout);
2750         KASSERT((fl->flags & FL_STARVING) == 0,
2751             ("%s: still starving", __func__));
2752 }
2753
2754 static int
2755 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2756     driver_intr_t *handler, void *arg, char *name)
2757 {
2758         int rc;
2759
2760         irq->rid = rid;
2761         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2762             RF_SHAREABLE | RF_ACTIVE);
2763         if (irq->res == NULL) {
2764                 device_printf(sc->dev,
2765                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2766                 return (ENOMEM);
2767         }
2768
2769         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2770             NULL, handler, arg, &irq->tag);
2771         if (rc != 0) {
2772                 device_printf(sc->dev,
2773                     "failed to setup interrupt for rid %d, name %s: %d\n",
2774                     rid, name, rc);
2775         } else if (name)
2776                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2777
2778         return (rc);
2779 }
2780
2781 static int
2782 t4_free_irq(struct adapter *sc, struct irq *irq)
2783 {
2784         if (irq->tag)
2785                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
2786         if (irq->res)
2787                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2788
2789         bzero(irq, sizeof(*irq));
2790
2791         return (0);
2792 }
2793
2794 static void
2795 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2796     unsigned int end)
2797 {
2798         uint32_t *p = (uint32_t *)(buf + start);
2799
2800         for ( ; start <= end; start += sizeof(uint32_t))
2801                 *p++ = t4_read_reg(sc, start);
2802 }
2803
2804 static void
2805 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2806 {
2807         int i;
2808         static const unsigned int reg_ranges[] = {
2809                 0x1008, 0x1108,
2810                 0x1180, 0x11b4,
2811                 0x11fc, 0x123c,
2812                 0x1300, 0x173c,
2813                 0x1800, 0x18fc,
2814                 0x3000, 0x30d8,
2815                 0x30e0, 0x5924,
2816                 0x5960, 0x59d4,
2817                 0x5a00, 0x5af8,
2818                 0x6000, 0x6098,
2819                 0x6100, 0x6150,
2820                 0x6200, 0x6208,
2821                 0x6240, 0x6248,
2822                 0x6280, 0x6338,
2823                 0x6370, 0x638c,
2824                 0x6400, 0x643c,
2825                 0x6500, 0x6524,
2826                 0x6a00, 0x6a38,
2827                 0x6a60, 0x6a78,
2828                 0x6b00, 0x6b84,
2829                 0x6bf0, 0x6c84,
2830                 0x6cf0, 0x6d84,
2831                 0x6df0, 0x6e84,
2832                 0x6ef0, 0x6f84,
2833                 0x6ff0, 0x7084,
2834                 0x70f0, 0x7184,
2835                 0x71f0, 0x7284,
2836                 0x72f0, 0x7384,
2837                 0x73f0, 0x7450,
2838                 0x7500, 0x7530,
2839                 0x7600, 0x761c,
2840                 0x7680, 0x76cc,
2841                 0x7700, 0x7798,
2842                 0x77c0, 0x77fc,
2843                 0x7900, 0x79fc,
2844                 0x7b00, 0x7c38,
2845                 0x7d00, 0x7efc,
2846                 0x8dc0, 0x8e1c,
2847                 0x8e30, 0x8e78,
2848                 0x8ea0, 0x8f6c,
2849                 0x8fc0, 0x9074,
2850                 0x90fc, 0x90fc,
2851                 0x9400, 0x9458,
2852                 0x9600, 0x96bc,
2853                 0x9800, 0x9808,
2854                 0x9820, 0x983c,
2855                 0x9850, 0x9864,
2856                 0x9c00, 0x9c6c,
2857                 0x9c80, 0x9cec,
2858                 0x9d00, 0x9d6c,
2859                 0x9d80, 0x9dec,
2860                 0x9e00, 0x9e6c,
2861                 0x9e80, 0x9eec,
2862                 0x9f00, 0x9f6c,
2863                 0x9f80, 0x9fec,
2864                 0xd004, 0xd03c,
2865                 0xdfc0, 0xdfe0,
2866                 0xe000, 0xea7c,
2867                 0xf000, 0x11190,
2868                 0x19040, 0x1906c,
2869                 0x19078, 0x19080,
2870                 0x1908c, 0x19124,
2871                 0x19150, 0x191b0,
2872                 0x191d0, 0x191e8,
2873                 0x19238, 0x1924c,
2874                 0x193f8, 0x19474,
2875                 0x19490, 0x194f8,
2876                 0x19800, 0x19f30,
2877                 0x1a000, 0x1a06c,
2878                 0x1a0b0, 0x1a120,
2879                 0x1a128, 0x1a138,
2880                 0x1a190, 0x1a1c4,
2881                 0x1a1fc, 0x1a1fc,
2882                 0x1e040, 0x1e04c,
2883                 0x1e284, 0x1e28c,
2884                 0x1e2c0, 0x1e2c0,
2885                 0x1e2e0, 0x1e2e0,
2886                 0x1e300, 0x1e384,
2887                 0x1e3c0, 0x1e3c8,
2888                 0x1e440, 0x1e44c,
2889                 0x1e684, 0x1e68c,
2890                 0x1e6c0, 0x1e6c0,
2891                 0x1e6e0, 0x1e6e0,
2892                 0x1e700, 0x1e784,
2893                 0x1e7c0, 0x1e7c8,
2894                 0x1e840, 0x1e84c,
2895                 0x1ea84, 0x1ea8c,
2896                 0x1eac0, 0x1eac0,
2897                 0x1eae0, 0x1eae0,
2898                 0x1eb00, 0x1eb84,
2899                 0x1ebc0, 0x1ebc8,
2900                 0x1ec40, 0x1ec4c,
2901                 0x1ee84, 0x1ee8c,
2902                 0x1eec0, 0x1eec0,
2903                 0x1eee0, 0x1eee0,
2904                 0x1ef00, 0x1ef84,
2905                 0x1efc0, 0x1efc8,
2906                 0x1f040, 0x1f04c,
2907                 0x1f284, 0x1f28c,
2908                 0x1f2c0, 0x1f2c0,
2909                 0x1f2e0, 0x1f2e0,
2910                 0x1f300, 0x1f384,
2911                 0x1f3c0, 0x1f3c8,
2912                 0x1f440, 0x1f44c,
2913                 0x1f684, 0x1f68c,
2914                 0x1f6c0, 0x1f6c0,
2915                 0x1f6e0, 0x1f6e0,
2916                 0x1f700, 0x1f784,
2917                 0x1f7c0, 0x1f7c8,
2918                 0x1f840, 0x1f84c,
2919                 0x1fa84, 0x1fa8c,
2920                 0x1fac0, 0x1fac0,
2921                 0x1fae0, 0x1fae0,
2922                 0x1fb00, 0x1fb84,
2923                 0x1fbc0, 0x1fbc8,
2924                 0x1fc40, 0x1fc4c,
2925                 0x1fe84, 0x1fe8c,
2926                 0x1fec0, 0x1fec0,
2927                 0x1fee0, 0x1fee0,
2928                 0x1ff00, 0x1ff84,
2929                 0x1ffc0, 0x1ffc8,
2930                 0x20000, 0x2002c,
2931                 0x20100, 0x2013c,
2932                 0x20190, 0x201c8,
2933                 0x20200, 0x20318,
2934                 0x20400, 0x20528,
2935                 0x20540, 0x20614,
2936                 0x21000, 0x21040,
2937                 0x2104c, 0x21060,
2938                 0x210c0, 0x210ec,
2939                 0x21200, 0x21268,
2940                 0x21270, 0x21284,
2941                 0x212fc, 0x21388,
2942                 0x21400, 0x21404,
2943                 0x21500, 0x21518,
2944                 0x2152c, 0x2153c,
2945                 0x21550, 0x21554,
2946                 0x21600, 0x21600,
2947                 0x21608, 0x21628,
2948                 0x21630, 0x2163c,
2949                 0x21700, 0x2171c,
2950                 0x21780, 0x2178c,
2951                 0x21800, 0x21c38,
2952                 0x21c80, 0x21d7c,
2953                 0x21e00, 0x21e04,
2954                 0x22000, 0x2202c,
2955                 0x22100, 0x2213c,
2956                 0x22190, 0x221c8,
2957                 0x22200, 0x22318,
2958                 0x22400, 0x22528,
2959                 0x22540, 0x22614,
2960                 0x23000, 0x23040,
2961                 0x2304c, 0x23060,
2962                 0x230c0, 0x230ec,
2963                 0x23200, 0x23268,
2964                 0x23270, 0x23284,
2965                 0x232fc, 0x23388,
2966                 0x23400, 0x23404,
2967                 0x23500, 0x23518,
2968                 0x2352c, 0x2353c,
2969                 0x23550, 0x23554,
2970                 0x23600, 0x23600,
2971                 0x23608, 0x23628,
2972                 0x23630, 0x2363c,
2973                 0x23700, 0x2371c,
2974                 0x23780, 0x2378c,
2975                 0x23800, 0x23c38,
2976                 0x23c80, 0x23d7c,
2977                 0x23e00, 0x23e04,
2978                 0x24000, 0x2402c,
2979                 0x24100, 0x2413c,
2980                 0x24190, 0x241c8,
2981                 0x24200, 0x24318,
2982                 0x24400, 0x24528,
2983                 0x24540, 0x24614,
2984                 0x25000, 0x25040,
2985                 0x2504c, 0x25060,
2986                 0x250c0, 0x250ec,
2987                 0x25200, 0x25268,
2988                 0x25270, 0x25284,
2989                 0x252fc, 0x25388,
2990                 0x25400, 0x25404,
2991                 0x25500, 0x25518,
2992                 0x2552c, 0x2553c,
2993                 0x25550, 0x25554,
2994                 0x25600, 0x25600,
2995                 0x25608, 0x25628,
2996                 0x25630, 0x2563c,
2997                 0x25700, 0x2571c,
2998                 0x25780, 0x2578c,
2999                 0x25800, 0x25c38,
3000                 0x25c80, 0x25d7c,
3001                 0x25e00, 0x25e04,
3002                 0x26000, 0x2602c,
3003                 0x26100, 0x2613c,
3004                 0x26190, 0x261c8,
3005                 0x26200, 0x26318,
3006                 0x26400, 0x26528,
3007                 0x26540, 0x26614,
3008                 0x27000, 0x27040,
3009                 0x2704c, 0x27060,
3010                 0x270c0, 0x270ec,
3011                 0x27200, 0x27268,
3012                 0x27270, 0x27284,
3013                 0x272fc, 0x27388,
3014                 0x27400, 0x27404,
3015                 0x27500, 0x27518,
3016                 0x2752c, 0x2753c,
3017                 0x27550, 0x27554,
3018                 0x27600, 0x27600,
3019                 0x27608, 0x27628,
3020                 0x27630, 0x2763c,
3021                 0x27700, 0x2771c,
3022                 0x27780, 0x2778c,
3023                 0x27800, 0x27c38,
3024                 0x27c80, 0x27d7c,
3025                 0x27e00, 0x27e04
3026         };
3027
3028         regs->version = 4 | (sc->params.rev << 10);
3029         for (i = 0; i < nitems(reg_ranges); i += 2)
3030                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
3031 }
3032
3033 static void
3034 cxgbe_tick(void *arg)
3035 {
3036         struct port_info *pi = arg;
3037         struct ifnet *ifp = pi->ifp;
3038         struct sge_txq *txq;
3039         int i, drops;
3040         struct port_stats *s = &pi->stats;
3041
3042         PORT_LOCK(pi);
3043         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3044                 PORT_UNLOCK(pi);
3045                 return; /* without scheduling another callout */
3046         }
3047
3048         t4_get_port_stats(pi->adapter, pi->tx_chan, s);
3049
3050         ifp->if_opackets = s->tx_frames - s->tx_pause;
3051         ifp->if_ipackets = s->rx_frames - s->rx_pause;
3052         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
3053         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
3054         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
3055         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
3056         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
3057             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
3058             s->rx_trunc3;
3059
3060         drops = s->tx_drop;
3061         for_each_txq(pi, i, txq)
3062                 drops += txq->br->br_drops;
3063         ifp->if_snd.ifq_drops = drops;
3064
3065         ifp->if_oerrors = s->tx_error_frames;
3066         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
3067             s->rx_fcs_err + s->rx_len_err;
3068
3069         callout_schedule(&pi->tick, hz);
3070         PORT_UNLOCK(pi);
3071 }
3072
3073 static void
3074 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
3075 {
3076         struct ifnet *vlan;
3077
3078         if (arg != ifp || ifp->if_type != IFT_ETHER)
3079                 return;
3080
3081         vlan = VLAN_DEVAT(ifp, vid);
3082         VLAN_SETCOOKIE(vlan, ifp);
3083 }
3084
3085 static int
3086 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
3087 {
3088
3089 #ifdef INVARIANTS
3090         panic("%s: opcode 0x%02x on iq %p with payload %p",
3091             __func__, rss->opcode, iq, m);
3092 #else
3093         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
3094             __func__, rss->opcode, iq, m);
3095         m_freem(m);
3096 #endif
3097         return (EDOOFUS);
3098 }
3099
3100 int
3101 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3102 {
3103         uintptr_t *loc, new;
3104
3105         if (opcode >= nitems(sc->cpl_handler))
3106                 return (EINVAL);
3107
3108         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3109         loc = (uintptr_t *) &sc->cpl_handler[opcode];
3110         atomic_store_rel_ptr(loc, new);
3111
3112         return (0);
3113 }
3114
3115 static int
3116 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
3117 {
3118
3119 #ifdef INVARIANTS
3120         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
3121 #else
3122         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
3123             __func__, iq, ctrl);
3124 #endif
3125         return (EDOOFUS);
3126 }
3127
3128 int
3129 t4_register_an_handler(struct adapter *sc, an_handler_t h)
3130 {
3131         uintptr_t *loc, new;
3132
3133         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
3134         loc = (uintptr_t *) &sc->an_handler;
3135         atomic_store_rel_ptr(loc, new);
3136
3137         return (0);
3138 }
3139
3140 static int
3141 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
3142 {
3143         const struct cpl_fw6_msg *cpl =
3144             __containerof(rpl, struct cpl_fw6_msg, data[0]);
3145
3146 #ifdef INVARIANTS
3147         panic("%s: fw_msg type %d", __func__, cpl->type);
3148 #else
3149         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
3150 #endif
3151         return (EDOOFUS);
3152 }
3153
3154 int
3155 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
3156 {
3157         uintptr_t *loc, new;
3158
3159         if (type >= nitems(sc->fw_msg_handler))
3160                 return (EINVAL);
3161
3162         /*
3163          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
3164          * handler dispatch table.  Reject any attempt to install a handler for
3165          * this subtype.
3166          */
3167         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
3168                 return (EINVAL);
3169
3170         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
3171         loc = (uintptr_t *) &sc->fw_msg_handler[type];
3172         atomic_store_rel_ptr(loc, new);
3173
3174         return (0);
3175 }
3176
3177 static int
3178 t4_sysctls(struct adapter *sc)
3179 {
3180         struct sysctl_ctx_list *ctx;
3181         struct sysctl_oid *oid;
3182         struct sysctl_oid_list *children, *c0;
3183         static char *caps[] = {
3184                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
3185                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",       /* caps[1] niccaps */
3186                 "\20\1TOE",                             /* caps[2] toecaps */
3187                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
3188                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
3189                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
3190                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
3191                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
3192         };
3193
3194         ctx = device_get_sysctl_ctx(sc->dev);
3195
3196         /*
3197          * dev.t4nex.X.
3198          */
3199         oid = device_get_sysctl_tree(sc->dev);
3200         c0 = children = SYSCTL_CHILDREN(oid);
3201
3202         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3203             &sc->params.nports, 0, "# of ports");
3204
3205         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3206             &sc->params.rev, 0, "chip hardware revision");
3207
3208         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3209             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3210
3211         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3212             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
3213
3214         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3215             &sc->cfcsum, 0, "config file checksum");
3216
3217         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3218             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3219             sysctl_bitfield, "A", "available link capabilities");
3220
3221         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3222             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3223             sysctl_bitfield, "A", "available NIC capabilities");
3224
3225         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3226             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3227             sysctl_bitfield, "A", "available TCP offload capabilities");
3228
3229         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3230             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3231             sysctl_bitfield, "A", "available RDMA capabilities");
3232
3233         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3234             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3235             sysctl_bitfield, "A", "available iSCSI capabilities");
3236
3237         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3238             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3239             sysctl_bitfield, "A", "available FCoE capabilities");
3240
3241         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3242             &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3243
3244         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3245             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3246             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3247             "interrupt holdoff timer values (us)");
3248
3249         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3250             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3251             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3252             "interrupt holdoff packet counter values");
3253
3254 #ifdef SBUF_DRAIN
3255         /*
3256          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3257          */
3258         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3259             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3260             "logs and miscellaneous information");
3261         children = SYSCTL_CHILDREN(oid);
3262
3263         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3264             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3265             sysctl_cctrl, "A", "congestion control");
3266
3267         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
3268             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3269             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
3270
3271         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
3272             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
3273             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
3274
3275         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
3276             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
3277             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
3278
3279         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
3280             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
3281             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
3282
3283         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
3284             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
3285             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
3286
3287         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
3288             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
3289             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
3290
3291         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
3292             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3293             sysctl_cim_la, "A", "CIM logic analyzer");
3294
3295         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
3296             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
3297             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
3298
3299         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
3300             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
3301             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
3302
3303         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
3304             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
3305             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
3306
3307         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
3308             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
3309             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
3310
3311         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
3312             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
3313             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
3314
3315         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
3316             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
3317             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
3318
3319         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
3320             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3321             sysctl_cim_qcfg, "A", "CIM queue configuration");
3322
3323         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3324             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3325             sysctl_cpl_stats, "A", "CPL statistics");
3326
3327         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3328             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3329             sysctl_ddp_stats, "A", "DDP statistics");
3330
3331         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3332             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3333             sysctl_devlog, "A", "firmware's device log");
3334
3335         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3336             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3337             sysctl_fcoe_stats, "A", "FCoE statistics");
3338
3339         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3340             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3341             sysctl_hw_sched, "A", "hardware scheduler ");
3342
3343         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3344             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3345             sysctl_l2t, "A", "hardware L2 table");
3346
3347         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3348             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3349             sysctl_lb_stats, "A", "loopback statistics");
3350
3351         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3352             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3353             sysctl_meminfo, "A", "memory regions");
3354
3355         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3356             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3357             sysctl_path_mtus, "A", "path MTUs");
3358
3359         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3360             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3361             sysctl_pm_stats, "A", "PM statistics");
3362
3363         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3364             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3365             sysctl_rdma_stats, "A", "RDMA statistics");
3366
3367         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3368             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3369             sysctl_tcp_stats, "A", "TCP statistics");
3370
3371         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3372             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3373             sysctl_tids, "A", "TID information");
3374
3375         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3376             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3377             sysctl_tp_err_stats, "A", "TP error statistics");
3378
3379         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3380             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3381             sysctl_tx_rate, "A", "Tx rate");
3382 #endif
3383
3384 #ifdef TCP_OFFLOAD
3385         if (is_offload(sc)) {
3386                 /*
3387                  * dev.t4nex.X.toe.
3388                  */
3389                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3390                     NULL, "TOE parameters");
3391                 children = SYSCTL_CHILDREN(oid);
3392
3393                 sc->tt.sndbuf = 256 * 1024;
3394                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3395                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
3396
3397                 sc->tt.ddp = 0;
3398                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3399                     &sc->tt.ddp, 0, "DDP allowed");
3400
3401                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
3402                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3403                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
3404
3405                 sc->tt.ddp_thres =
3406                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
3407                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3408                     &sc->tt.ddp_thres, 0, "DDP threshold");
3409         }
3410 #endif
3411
3412
3413         return (0);
3414 }
3415
3416 static int
3417 cxgbe_sysctls(struct port_info *pi)
3418 {
3419         struct sysctl_ctx_list *ctx;
3420         struct sysctl_oid *oid;
3421         struct sysctl_oid_list *children;
3422
3423         ctx = device_get_sysctl_ctx(pi->dev);
3424
3425         /*
3426          * dev.cxgbe.X.
3427          */
3428         oid = device_get_sysctl_tree(pi->dev);
3429         children = SYSCTL_CHILDREN(oid);
3430
3431         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3432             &pi->nrxq, 0, "# of rx queues");
3433         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3434             &pi->ntxq, 0, "# of tx queues");
3435         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3436             &pi->first_rxq, 0, "index of first rx queue");
3437         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3438             &pi->first_txq, 0, "index of first tx queue");
3439
3440 #ifdef TCP_OFFLOAD
3441         if (is_offload(pi->adapter)) {
3442                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3443                     &pi->nofldrxq, 0,
3444                     "# of rx queues for offloaded TCP connections");
3445                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3446                     &pi->nofldtxq, 0,
3447                     "# of tx queues for offloaded TCP connections");
3448                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3449                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3450                     "index of first TOE rx queue");
3451                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3452                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
3453                     "index of first TOE tx queue");
3454         }
3455 #endif
3456
3457         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3458             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3459             "holdoff timer index");
3460         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3461             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3462             "holdoff packet counter index");
3463
3464         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3465             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3466             "rx queue size");
3467         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3468             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3469             "tx queue size");
3470
3471         /*
3472          * dev.cxgbe.X.stats.
3473          */
3474         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3475             NULL, "port statistics");
3476         children = SYSCTL_CHILDREN(oid);
3477
3478 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3479         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3480             CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3481             sysctl_handle_t4_reg64, "QU", desc)
3482
3483         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3484             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3485         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3486             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3487         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3488             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3489         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3490             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3491         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3492             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3493         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3494             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3495         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3496             "# of tx frames in this range",
3497             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3498         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3499             "# of tx frames in this range",
3500             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3501         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3502             "# of tx frames in this range",
3503             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3504         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3505             "# of tx frames in this range",
3506             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3507         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3508             "# of tx frames in this range",
3509             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3510         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3511             "# of tx frames in this range",
3512             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3513         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3514             "# of tx frames in this range",
3515             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3516         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3517             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3518         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3519             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3520         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3521             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3522         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3523             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3524         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3525             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3526         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3527             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3528         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3529             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3530         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3531             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3532         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3533             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3534         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3535             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3536
3537         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3538             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3539         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3540             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3541         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3542             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3543         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3544             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3545         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3546             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3547         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3548             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3549         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3550             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3551         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3552             "# of frames received with bad FCS",
3553             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3554         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3555             "# of frames received with length error",
3556             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3557         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3558             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3559         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3560             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3561         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3562             "# of rx frames in this range",
3563             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3564         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3565             "# of rx frames in this range",
3566             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3567         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3568             "# of rx frames in this range",
3569             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3570         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3571             "# of rx frames in this range",
3572             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3573         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3574             "# of rx frames in this range",
3575             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3576         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3577             "# of rx frames in this range",
3578             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3579         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3580             "# of rx frames in this range",
3581             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3582         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3583             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3584         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3585             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3586         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3587             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3588         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3589             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3590         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3591             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3592         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3593             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3594         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3595             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3596         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3597             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3598         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3599             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3600
3601 #undef SYSCTL_ADD_T4_REG64
3602
3603 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3604         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3605             &pi->stats.name, desc)
3606
3607         /* We get these from port_stats and they may be stale by upto 1s */
3608         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3609             "# drops due to buffer-group 0 overflows");
3610         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3611             "# drops due to buffer-group 1 overflows");
3612         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3613             "# drops due to buffer-group 2 overflows");
3614         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3615             "# drops due to buffer-group 3 overflows");
3616         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3617             "# of buffer-group 0 truncated packets");
3618         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3619             "# of buffer-group 1 truncated packets");
3620         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3621             "# of buffer-group 2 truncated packets");
3622         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3623             "# of buffer-group 3 truncated packets");
3624
3625 #undef SYSCTL_ADD_T4_PORTSTAT
3626
3627         return (0);
3628 }
3629
3630 static int
3631 sysctl_int_array(SYSCTL_HANDLER_ARGS)
3632 {
3633         int rc, *i;
3634         struct sbuf sb;
3635
3636         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3637         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3638                 sbuf_printf(&sb, "%d ", *i);
3639         sbuf_trim(&sb);
3640         sbuf_finish(&sb);
3641         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3642         sbuf_delete(&sb);
3643         return (rc);
3644 }
3645
3646 static int
3647 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
3648 {
3649         int rc;
3650         struct sbuf *sb;
3651
3652         rc = sysctl_wire_old_buffer(req, 0);
3653         if (rc != 0)
3654                 return(rc);
3655
3656         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3657         if (sb == NULL)
3658                 return (ENOMEM);
3659
3660         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3661         rc = sbuf_finish(sb);
3662         sbuf_delete(sb);
3663
3664         return (rc);
3665 }
3666
3667 static int
3668 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3669 {
3670         struct port_info *pi = arg1;
3671         struct adapter *sc = pi->adapter;
3672         int idx, rc, i;
3673         struct sge_rxq *rxq;
3674         uint8_t v;
3675
3676         idx = pi->tmr_idx;
3677
3678         rc = sysctl_handle_int(oidp, &idx, 0, req);
3679         if (rc != 0 || req->newptr == NULL)
3680                 return (rc);
3681
3682         if (idx < 0 || idx >= SGE_NTIMERS)
3683                 return (EINVAL);
3684
3685         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3686             "t4tmr");
3687         if (rc)
3688                 return (rc);
3689
3690         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3691         for_each_rxq(pi, i, rxq) {
3692 #ifdef atomic_store_rel_8
3693                 atomic_store_rel_8(&rxq->iq.intr_params, v);
3694 #else
3695                 rxq->iq.intr_params = v;
3696 #endif
3697         }
3698         pi->tmr_idx = idx;
3699
3700         end_synchronized_op(sc, LOCK_HELD);
3701         return (0);
3702 }
3703
3704 static int
3705 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3706 {
3707         struct port_info *pi = arg1;
3708         struct adapter *sc = pi->adapter;
3709         int idx, rc;
3710
3711         idx = pi->pktc_idx;
3712
3713         rc = sysctl_handle_int(oidp, &idx, 0, req);
3714         if (rc != 0 || req->newptr == NULL)
3715                 return (rc);
3716
3717         if (idx < -1 || idx >= SGE_NCOUNTERS)
3718                 return (EINVAL);
3719
3720         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3721             "t4pktc");
3722         if (rc)
3723                 return (rc);
3724
3725         if (pi->flags & PORT_INIT_DONE)
3726                 rc = EBUSY; /* cannot be changed once the queues are created */
3727         else
3728                 pi->pktc_idx = idx;
3729
3730         end_synchronized_op(sc, LOCK_HELD);
3731         return (rc);
3732 }
3733
3734 static int
3735 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3736 {
3737         struct port_info *pi = arg1;
3738         struct adapter *sc = pi->adapter;
3739         int qsize, rc;
3740
3741         qsize = pi->qsize_rxq;
3742
3743         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3744         if (rc != 0 || req->newptr == NULL)
3745                 return (rc);
3746
3747         if (qsize < 128 || (qsize & 7))
3748                 return (EINVAL);
3749
3750         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3751             "t4rxqs");
3752         if (rc)
3753                 return (rc);
3754
3755         if (pi->flags & PORT_INIT_DONE)
3756                 rc = EBUSY; /* cannot be changed once the queues are created */
3757         else
3758                 pi->qsize_rxq = qsize;
3759
3760         end_synchronized_op(sc, LOCK_HELD);
3761         return (rc);
3762 }
3763
3764 static int
3765 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3766 {
3767         struct port_info *pi = arg1;
3768         struct adapter *sc = pi->adapter;
3769         int qsize, rc;
3770
3771         qsize = pi->qsize_txq;
3772
3773         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3774         if (rc != 0 || req->newptr == NULL)
3775                 return (rc);
3776
3777         /* bufring size must be powerof2 */
3778         if (qsize < 128 || !powerof2(qsize))
3779                 return (EINVAL);
3780
3781         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3782             "t4txqs");
3783         if (rc)
3784                 return (rc);
3785
3786         if (pi->flags & PORT_INIT_DONE)
3787                 rc = EBUSY; /* cannot be changed once the queues are created */
3788         else
3789                 pi->qsize_txq = qsize;
3790
3791         end_synchronized_op(sc, LOCK_HELD);
3792         return (rc);
3793 }
3794
3795 static int
3796 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3797 {
3798         struct adapter *sc = arg1;
3799         int reg = arg2;
3800         uint64_t val;
3801
3802         val = t4_read_reg64(sc, reg);
3803
3804         return (sysctl_handle_64(oidp, &val, 0, req));
3805 }
3806
3807 #ifdef SBUF_DRAIN
3808 static int
3809 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
3810 {
3811         struct adapter *sc = arg1;
3812         struct sbuf *sb;
3813         int rc, i;
3814         uint16_t incr[NMTUS][NCCTRL_WIN];
3815         static const char *dec_fac[] = {
3816                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3817                 "0.9375"
3818         };
3819
3820         rc = sysctl_wire_old_buffer(req, 0);
3821         if (rc != 0)
3822                 return (rc);
3823
3824         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3825         if (sb == NULL)
3826                 return (ENOMEM);
3827
3828         t4_read_cong_tbl(sc, incr);
3829
3830         for (i = 0; i < NCCTRL_WIN; ++i) {
3831                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3832                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3833                     incr[5][i], incr[6][i], incr[7][i]);
3834                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3835                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3836                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3837                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3838         }
3839
3840         rc = sbuf_finish(sb);
3841         sbuf_delete(sb);
3842
3843         return (rc);
3844 }
3845
3846 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ] = {
3847         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
3848         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI"  /* obq's */
3849 };
3850
3851 static int
3852 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
3853 {
3854         struct adapter *sc = arg1;
3855         struct sbuf *sb;
3856         int rc, i, n, qid = arg2;
3857         uint32_t *buf, *p;
3858         char *qtype;
3859
3860         KASSERT(qid >= 0 && qid < nitems(qname),
3861             ("%s: bad qid %d\n", __func__, qid));
3862
3863         if (qid < CIM_NUM_IBQ) {
3864                 /* inbound queue */
3865                 qtype = "IBQ";
3866                 n = 4 * CIM_IBQ_SIZE;
3867                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
3868                 rc = t4_read_cim_ibq(sc, qid, buf, n);
3869         } else {
3870                 /* outbound queue */
3871                 qtype = "OBQ";
3872                 qid -= CIM_NUM_IBQ;
3873                 n = 4 * 6 * CIM_OBQ_SIZE;
3874                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
3875                 rc = t4_read_cim_obq(sc, qid, buf, n);
3876         }
3877
3878         if (rc < 0) {
3879                 rc = -rc;
3880                 goto done;
3881         }
3882         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
3883
3884         rc = sysctl_wire_old_buffer(req, 0);
3885         if (rc != 0)
3886                 goto done;
3887
3888         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3889         if (sb == NULL) {
3890                 rc = ENOMEM;
3891                 goto done;
3892         }
3893
3894         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
3895         for (i = 0, p = buf; i < n; i += 16, p += 4)
3896                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
3897                     p[2], p[3]);
3898
3899         rc = sbuf_finish(sb);
3900         sbuf_delete(sb);
3901 done:
3902         free(buf, M_CXGBE);
3903         return (rc);
3904 }
3905
3906 static int
3907 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
3908 {
3909         struct adapter *sc = arg1;
3910         u_int cfg;
3911         struct sbuf *sb;
3912         uint32_t *buf, *p;
3913         int rc;
3914
3915         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
3916         if (rc != 0)
3917                 return (rc);
3918
3919         rc = sysctl_wire_old_buffer(req, 0);
3920         if (rc != 0)
3921                 return (rc);
3922
3923         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3924         if (sb == NULL)
3925                 return (ENOMEM);
3926
3927         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
3928             M_ZERO | M_WAITOK);
3929
3930         rc = -t4_cim_read_la(sc, buf, NULL);
3931         if (rc != 0)
3932                 goto done;
3933
3934         sbuf_printf(sb, "Status   Data      PC%s",
3935             cfg & F_UPDBGLACAPTPCONLY ? "" :
3936             "     LS0Stat  LS0Addr             LS0Data");
3937
3938         KASSERT((sc->params.cim_la_size & 7) == 0,
3939             ("%s: p will walk off the end of buf", __func__));
3940
3941         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
3942                 if (cfg & F_UPDBGLACAPTPCONLY) {
3943                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
3944                             p[6], p[7]);
3945                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
3946                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
3947                             p[4] & 0xff, p[5] >> 8);
3948                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
3949                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
3950                             p[1] & 0xf, p[2] >> 4);
3951                 } else {
3952                         sbuf_printf(sb,
3953                             "\n  %02x   %x%07x %x%07x %08x %08x "
3954                             "%08x%08x%08x%08x",
3955                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
3956                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
3957                             p[6], p[7]);
3958                 }
3959         }
3960
3961         rc = sbuf_finish(sb);
3962         sbuf_delete(sb);
3963 done:
3964         free(buf, M_CXGBE);
3965         return (rc);
3966 }
3967
3968 static int
3969 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
3970 {
3971         struct adapter *sc = arg1;
3972         struct sbuf *sb;
3973         int rc, i;
3974         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ];
3975         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ];
3976         uint16_t thres[CIM_NUM_IBQ];
3977         uint32_t obq_wr[2 * CIM_NUM_OBQ], *wr = obq_wr;
3978         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ)], *p = stat;
3979
3980         rc = -t4_cim_read(sc, A_UP_IBQ_0_RDADDR, nitems(stat), stat);
3981         if (rc == 0)
3982                 rc = -t4_cim_read(sc, A_UP_OBQ_0_REALADDR, nitems(obq_wr),
3983                     obq_wr);
3984         if (rc != 0)
3985                 return (rc);
3986
3987         t4_read_cimq_cfg(sc, base, size, thres);
3988
3989         rc = sysctl_wire_old_buffer(req, 0);
3990         if (rc != 0)
3991                 return (rc);
3992
3993         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3994         if (sb == NULL)
3995                 return (ENOMEM);
3996
3997         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
3998
3999         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
4000                 sbuf_printf(sb, "\n%5s %5x %5u %4u %6x  %4x %4u %4u %5u",
4001                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
4002                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
4003                     G_QUEREMFLITS(p[2]) * 16);
4004         for ( ; i < CIM_NUM_IBQ + CIM_NUM_OBQ; i++, p += 4, wr += 2)
4005                 sbuf_printf(sb, "\n%5s %5x %5u %11x  %4x %4u %4u %5u", qname[i],
4006                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
4007                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
4008                     G_QUEREMFLITS(p[2]) * 16);
4009
4010         rc = sbuf_finish(sb);
4011         sbuf_delete(sb);
4012
4013         return (rc);
4014 }
4015
4016 static int
4017 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
4018 {
4019         struct adapter *sc = arg1;
4020         struct sbuf *sb;
4021         int rc;
4022         struct tp_cpl_stats stats;
4023
4024         rc = sysctl_wire_old_buffer(req, 0);
4025         if (rc != 0)
4026                 return (rc);
4027
4028         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4029         if (sb == NULL)
4030                 return (ENOMEM);
4031
4032         t4_tp_get_cpl_stats(sc, &stats);
4033
4034         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4035             "channel 3\n");
4036         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
4037                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
4038         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
4039                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
4040
4041         rc = sbuf_finish(sb);
4042         sbuf_delete(sb);
4043
4044         return (rc);
4045 }
4046
4047 static int
4048 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
4049 {
4050         struct adapter *sc = arg1;
4051         struct sbuf *sb;
4052         int rc;
4053         struct tp_usm_stats stats;
4054
4055         rc = sysctl_wire_old_buffer(req, 0);
4056         if (rc != 0)
4057                 return(rc);
4058
4059         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4060         if (sb == NULL)
4061                 return (ENOMEM);
4062
4063         t4_get_usm_stats(sc, &stats);
4064
4065         sbuf_printf(sb, "Frames: %u\n", stats.frames);
4066         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
4067         sbuf_printf(sb, "Drops:  %u", stats.drops);
4068
4069         rc = sbuf_finish(sb);
4070         sbuf_delete(sb);
4071
4072         return (rc);
4073 }
4074
4075 const char *devlog_level_strings[] = {
4076         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
4077         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
4078         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
4079         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
4080         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
4081         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
4082 };
4083
4084 const char *devlog_facility_strings[] = {
4085         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
4086         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
4087         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
4088         [FW_DEVLOG_FACILITY_RES]        = "RES",
4089         [FW_DEVLOG_FACILITY_HW]         = "HW",
4090         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
4091         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
4092         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
4093         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
4094         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
4095         [FW_DEVLOG_FACILITY_VI]         = "VI",
4096         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
4097         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
4098         [FW_DEVLOG_FACILITY_TM]         = "TM",
4099         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
4100         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
4101         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
4102         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
4103         [FW_DEVLOG_FACILITY_RI]         = "RI",
4104         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
4105         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
4106         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
4107         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
4108 };
4109
4110 static int
4111 sysctl_devlog(SYSCTL_HANDLER_ARGS)
4112 {
4113         struct adapter *sc = arg1;
4114         struct devlog_params *dparams = &sc->params.devlog;
4115         struct fw_devlog_e *buf, *e;
4116         int i, j, rc, nentries, first = 0;
4117         struct sbuf *sb;
4118         uint64_t ftstamp = UINT64_MAX;
4119
4120         if (dparams->start == 0)
4121                 return (ENXIO);
4122
4123         nentries = dparams->size / sizeof(struct fw_devlog_e);
4124
4125         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
4126         if (buf == NULL)
4127                 return (ENOMEM);
4128
4129         rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
4130             (void *)buf);
4131         if (rc != 0)
4132                 goto done;
4133
4134         for (i = 0; i < nentries; i++) {
4135                 e = &buf[i];
4136
4137                 if (e->timestamp == 0)
4138                         break;  /* end */
4139
4140                 e->timestamp = be64toh(e->timestamp);
4141                 e->seqno = be32toh(e->seqno);
4142                 for (j = 0; j < 8; j++)
4143                         e->params[j] = be32toh(e->params[j]);
4144
4145                 if (e->timestamp < ftstamp) {
4146                         ftstamp = e->timestamp;
4147                         first = i;
4148                 }
4149         }
4150
4151         if (buf[first].timestamp == 0)
4152                 goto done;      /* nothing in the log */
4153
4154         rc = sysctl_wire_old_buffer(req, 0);
4155         if (rc != 0)
4156                 goto done;
4157
4158         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4159         if (sb == NULL) {
4160                 rc = ENOMEM;
4161                 goto done;
4162         }
4163         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
4164             "Seq#", "Tstamp", "Level", "Facility", "Message");
4165
4166         i = first;
4167         do {
4168                 e = &buf[i];
4169                 if (e->timestamp == 0)
4170                         break;  /* end */
4171
4172                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
4173                     e->seqno, e->timestamp,
4174                     (e->level < nitems(devlog_level_strings) ?
4175                         devlog_level_strings[e->level] : "UNKNOWN"),
4176                     (e->facility < nitems(devlog_facility_strings) ?
4177                         devlog_facility_strings[e->facility] : "UNKNOWN"));
4178                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
4179                     e->params[2], e->params[3], e->params[4],
4180                     e->params[5], e->params[6], e->params[7]);
4181
4182                 if (++i == nentries)
4183                         i = 0;
4184         } while (i != first);
4185
4186         rc = sbuf_finish(sb);
4187         sbuf_delete(sb);
4188 done:
4189         free(buf, M_CXGBE);
4190         return (rc);
4191 }
4192
4193 static int
4194 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
4195 {
4196         struct adapter *sc = arg1;
4197         struct sbuf *sb;
4198         int rc;
4199         struct tp_fcoe_stats stats[4];
4200
4201         rc = sysctl_wire_old_buffer(req, 0);
4202         if (rc != 0)
4203                 return (rc);
4204
4205         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4206         if (sb == NULL)
4207                 return (ENOMEM);
4208
4209         t4_get_fcoe_stats(sc, 0, &stats[0]);
4210         t4_get_fcoe_stats(sc, 1, &stats[1]);
4211         t4_get_fcoe_stats(sc, 2, &stats[2]);
4212         t4_get_fcoe_stats(sc, 3, &stats[3]);
4213
4214         sbuf_printf(sb, "                   channel 0        channel 1        "
4215             "channel 2        channel 3\n");
4216         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
4217             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
4218             stats[3].octetsDDP);
4219         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
4220             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
4221         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
4222             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
4223             stats[3].framesDrop);
4224
4225         rc = sbuf_finish(sb);
4226         sbuf_delete(sb);
4227
4228         return (rc);
4229 }
4230
4231 static int
4232 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
4233 {
4234         struct adapter *sc = arg1;
4235         struct sbuf *sb;
4236         int rc, i;
4237         unsigned int map, kbps, ipg, mode;
4238         unsigned int pace_tab[NTX_SCHED];
4239
4240         rc = sysctl_wire_old_buffer(req, 0);
4241         if (rc != 0)
4242                 return (rc);
4243
4244         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4245         if (sb == NULL)
4246                 return (ENOMEM);
4247
4248         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
4249         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
4250         t4_read_pace_tbl(sc, pace_tab);
4251
4252         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
4253             "Class IPG (0.1 ns)   Flow IPG (us)");
4254
4255         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
4256                 t4_get_tx_sched(sc, i, &kbps, &ipg);
4257                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
4258                     (mode & (1 << i)) ? "flow" : "class", map & 3);
4259                 if (kbps)
4260                         sbuf_printf(sb, "%9u     ", kbps);
4261                 else
4262                         sbuf_printf(sb, " disabled     ");
4263
4264                 if (ipg)
4265                         sbuf_printf(sb, "%13u        ", ipg);
4266                 else
4267                         sbuf_printf(sb, "     disabled        ");
4268
4269                 if (pace_tab[i])
4270                         sbuf_printf(sb, "%10u", pace_tab[i]);
4271                 else
4272                         sbuf_printf(sb, "  disabled");
4273         }
4274
4275         rc = sbuf_finish(sb);
4276         sbuf_delete(sb);
4277
4278         return (rc);
4279 }
4280
4281 static int
4282 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
4283 {
4284         struct adapter *sc = arg1;
4285         struct sbuf *sb;
4286         int rc, i, j;
4287         uint64_t *p0, *p1;
4288         struct lb_port_stats s[2];
4289         static const char *stat_name[] = {
4290                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
4291                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
4292                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
4293                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
4294                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
4295                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
4296                 "BG2FramesTrunc:", "BG3FramesTrunc:"
4297         };
4298
4299         rc = sysctl_wire_old_buffer(req, 0);
4300         if (rc != 0)
4301                 return (rc);
4302
4303         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4304         if (sb == NULL)
4305                 return (ENOMEM);
4306
4307         memset(s, 0, sizeof(s));
4308
4309         for (i = 0; i < 4; i += 2) {
4310                 t4_get_lb_stats(sc, i, &s[0]);
4311                 t4_get_lb_stats(sc, i + 1, &s[1]);
4312
4313                 p0 = &s[0].octets;
4314                 p1 = &s[1].octets;
4315                 sbuf_printf(sb, "%s                       Loopback %u"
4316                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
4317
4318                 for (j = 0; j < nitems(stat_name); j++)
4319                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
4320                                    *p0++, *p1++);
4321         }
4322
4323         rc = sbuf_finish(sb);
4324         sbuf_delete(sb);
4325
4326         return (rc);
4327 }
4328
4329 struct mem_desc {
4330         unsigned int base;
4331         unsigned int limit;
4332         unsigned int idx;
4333 };
4334
4335 static int
4336 mem_desc_cmp(const void *a, const void *b)
4337 {
4338         return ((const struct mem_desc *)a)->base -
4339                ((const struct mem_desc *)b)->base;
4340 }
4341
4342 static void
4343 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
4344     unsigned int to)
4345 {
4346         unsigned int size;
4347
4348         size = to - from + 1;
4349         if (size == 0)
4350                 return;
4351
4352         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
4353         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
4354 }
4355
4356 static int
4357 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
4358 {
4359         struct adapter *sc = arg1;
4360         struct sbuf *sb;
4361         int rc, i, n;
4362         uint32_t lo, hi;
4363         static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
4364         static const char *region[] = {
4365                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
4366                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
4367                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
4368                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
4369                 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
4370                 "ULPTX state:", "On-chip queues:"
4371         };
4372         struct mem_desc avail[3];
4373         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
4374         struct mem_desc *md = mem;
4375
4376         rc = sysctl_wire_old_buffer(req, 0);
4377         if (rc != 0)
4378                 return (rc);
4379
4380         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4381         if (sb == NULL)
4382                 return (ENOMEM);
4383
4384         for (i = 0; i < nitems(mem); i++) {
4385                 mem[i].limit = 0;
4386                 mem[i].idx = i;
4387         }
4388
4389         /* Find and sort the populated memory ranges */
4390         i = 0;
4391         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4392         if (lo & F_EDRAM0_ENABLE) {
4393                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4394                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
4395                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
4396                 avail[i].idx = 0;
4397                 i++;
4398         }
4399         if (lo & F_EDRAM1_ENABLE) {
4400                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4401                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
4402                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
4403                 avail[i].idx = 1;
4404                 i++;
4405         }
4406         if (lo & F_EXT_MEM_ENABLE) {
4407                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4408                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
4409                 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
4410                 avail[i].idx = 2;
4411                 i++;
4412         }
4413         if (!i)                                    /* no memory available */
4414                 return 0;
4415         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
4416
4417         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
4418         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
4419         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
4420         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4421         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
4422         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
4423         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
4424         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
4425         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
4426
4427         /* the next few have explicit upper bounds */
4428         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
4429         md->limit = md->base - 1 +
4430                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
4431                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
4432         md++;
4433
4434         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4435         md->limit = md->base - 1 +
4436                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4437                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4438         md++;
4439
4440         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4441                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4442                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4443                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4444         } else {
4445                 md->base = 0;
4446                 md->idx = nitems(region);  /* hide it */
4447         }
4448         md++;
4449
4450 #define ulp_region(reg) \
4451         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4452         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4453
4454         ulp_region(RX_ISCSI);
4455         ulp_region(RX_TDDP);
4456         ulp_region(TX_TPT);
4457         ulp_region(RX_STAG);
4458         ulp_region(RX_RQ);
4459         ulp_region(RX_RQUDP);
4460         ulp_region(RX_PBL);
4461         ulp_region(TX_PBL);
4462 #undef ulp_region
4463
4464         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4465         md->limit = md->base + sc->tids.ntids - 1;
4466         md++;
4467         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4468         md->limit = md->base + sc->tids.ntids - 1;
4469         md++;
4470
4471         md->base = sc->vres.ocq.start;
4472         if (sc->vres.ocq.size)
4473                 md->limit = md->base + sc->vres.ocq.size - 1;
4474         else
4475                 md->idx = nitems(region);  /* hide it */
4476         md++;
4477
4478         /* add any address-space holes, there can be up to 3 */
4479         for (n = 0; n < i - 1; n++)
4480                 if (avail[n].limit < avail[n + 1].base)
4481                         (md++)->base = avail[n].limit;
4482         if (avail[n].limit)
4483                 (md++)->base = avail[n].limit;
4484
4485         n = md - mem;
4486         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4487
4488         for (lo = 0; lo < i; lo++)
4489                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4490                                 avail[lo].limit - 1);
4491
4492         sbuf_printf(sb, "\n");
4493         for (i = 0; i < n; i++) {
4494                 if (mem[i].idx >= nitems(region))
4495                         continue;                        /* skip holes */
4496                 if (!mem[i].limit)
4497                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4498                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
4499                                 mem[i].limit);
4500         }
4501
4502         sbuf_printf(sb, "\n");
4503         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4504         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4505         mem_region_show(sb, "uP RAM:", lo, hi);
4506
4507         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4508         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4509         mem_region_show(sb, "uP Extmem2:", lo, hi);
4510
4511         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4512         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4513                    G_PMRXMAXPAGE(lo),
4514                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4515                    (lo & F_PMRXNUMCHN) ? 2 : 1);
4516
4517         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4518         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4519         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4520                    G_PMTXMAXPAGE(lo),
4521                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4522                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4523         sbuf_printf(sb, "%u p-structs\n",
4524                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4525
4526         for (i = 0; i < 4; i++) {
4527                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4528                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4529                            i, G_USED(lo), G_ALLOC(lo));
4530         }
4531         for (i = 0; i < 4; i++) {
4532                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4533                 sbuf_printf(sb,
4534                            "\nLoopback %d using %u pages out of %u allocated",
4535                            i, G_USED(lo), G_ALLOC(lo));
4536         }
4537
4538         rc = sbuf_finish(sb);
4539         sbuf_delete(sb);
4540
4541         return (rc);
4542 }
4543
4544 static int
4545 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4546 {
4547         struct adapter *sc = arg1;
4548         struct sbuf *sb;
4549         int rc;
4550         uint16_t mtus[NMTUS];
4551
4552         rc = sysctl_wire_old_buffer(req, 0);
4553         if (rc != 0)
4554                 return (rc);
4555
4556         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4557         if (sb == NULL)
4558                 return (ENOMEM);
4559
4560         t4_read_mtu_tbl(sc, mtus, NULL);
4561
4562         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4563             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4564             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4565             mtus[14], mtus[15]);
4566
4567         rc = sbuf_finish(sb);
4568         sbuf_delete(sb);
4569
4570         return (rc);
4571 }
4572
4573 static int
4574 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4575 {
4576         struct adapter *sc = arg1;
4577         struct sbuf *sb;
4578         int rc, i;
4579         uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4580         uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4581         static const char *pm_stats[] = {
4582                 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4583         };
4584
4585         rc = sysctl_wire_old_buffer(req, 0);
4586         if (rc != 0)
4587                 return (rc);
4588
4589         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4590         if (sb == NULL)
4591                 return (ENOMEM);
4592
4593         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4594         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4595
4596         sbuf_printf(sb, "                Tx count            Tx cycles    "
4597             "Rx count            Rx cycles");
4598         for (i = 0; i < PM_NSTATS; i++)
4599                 sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4600                     pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4601
4602         rc = sbuf_finish(sb);
4603         sbuf_delete(sb);
4604
4605         return (rc);
4606 }
4607
4608 static int
4609 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4610 {
4611         struct adapter *sc = arg1;
4612         struct sbuf *sb;
4613         int rc;
4614         struct tp_rdma_stats stats;
4615
4616         rc = sysctl_wire_old_buffer(req, 0);
4617         if (rc != 0)
4618                 return (rc);
4619
4620         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4621         if (sb == NULL)
4622                 return (ENOMEM);
4623
4624         t4_tp_get_rdma_stats(sc, &stats);
4625         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4626         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4627
4628         rc = sbuf_finish(sb);
4629         sbuf_delete(sb);
4630
4631         return (rc);
4632 }
4633
4634 static int
4635 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4636 {
4637         struct adapter *sc = arg1;
4638         struct sbuf *sb;
4639         int rc;
4640         struct tp_tcp_stats v4, v6;
4641
4642         rc = sysctl_wire_old_buffer(req, 0);
4643         if (rc != 0)
4644                 return (rc);
4645
4646         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4647         if (sb == NULL)
4648                 return (ENOMEM);
4649
4650         t4_tp_get_tcp_stats(sc, &v4, &v6);
4651         sbuf_printf(sb,
4652             "                                IP                 IPv6\n");
4653         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4654             v4.tcpOutRsts, v6.tcpOutRsts);
4655         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4656             v4.tcpInSegs, v6.tcpInSegs);
4657         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4658             v4.tcpOutSegs, v6.tcpOutSegs);
4659         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4660             v4.tcpRetransSegs, v6.tcpRetransSegs);
4661
4662         rc = sbuf_finish(sb);
4663         sbuf_delete(sb);
4664
4665         return (rc);
4666 }
4667
4668 static int
4669 sysctl_tids(SYSCTL_HANDLER_ARGS)
4670 {
4671         struct adapter *sc = arg1;
4672         struct sbuf *sb;
4673         int rc;
4674         struct tid_info *t = &sc->tids;
4675
4676         rc = sysctl_wire_old_buffer(req, 0);
4677         if (rc != 0)
4678                 return (rc);
4679
4680         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4681         if (sb == NULL)
4682                 return (ENOMEM);
4683
4684         if (t->natids) {
4685                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4686                     t->atids_in_use);
4687         }
4688
4689         if (t->ntids) {
4690                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4691                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4692
4693                         if (b) {
4694                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4695                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4696                                     t->ntids - 1);
4697                         } else {
4698                                 sbuf_printf(sb, "TID range: %u-%u",
4699                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4700                                     t->ntids - 1);
4701                         }
4702                 } else
4703                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4704                 sbuf_printf(sb, ", in use: %u\n",
4705                     atomic_load_acq_int(&t->tids_in_use));
4706         }
4707
4708         if (t->nstids) {
4709                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4710                     t->stid_base + t->nstids - 1, t->stids_in_use);
4711         }
4712
4713         if (t->nftids) {
4714                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4715                     t->ftid_base + t->nftids - 1);
4716         }
4717
4718         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4719             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4720             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4721
4722         rc = sbuf_finish(sb);
4723         sbuf_delete(sb);
4724
4725         return (rc);
4726 }
4727
4728 static int
4729 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4730 {
4731         struct adapter *sc = arg1;
4732         struct sbuf *sb;
4733         int rc;
4734         struct tp_err_stats stats;
4735
4736         rc = sysctl_wire_old_buffer(req, 0);
4737         if (rc != 0)
4738                 return (rc);
4739
4740         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4741         if (sb == NULL)
4742                 return (ENOMEM);
4743
4744         t4_tp_get_err_stats(sc, &stats);
4745
4746         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4747                       "channel 3\n");
4748         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4749             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4750             stats.macInErrs[3]);
4751         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4752             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4753             stats.hdrInErrs[3]);
4754         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4755             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4756             stats.tcpInErrs[3]);
4757         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4758             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4759             stats.tcp6InErrs[3]);
4760         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4761             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4762             stats.tnlCongDrops[3]);
4763         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4764             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4765             stats.tnlTxDrops[3]);
4766         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4767             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4768             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4769         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4770             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4771             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4772         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4773             stats.ofldNoNeigh, stats.ofldCongDefer);
4774
4775         rc = sbuf_finish(sb);
4776         sbuf_delete(sb);
4777
4778         return (rc);
4779 }
4780
4781 static int
4782 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4783 {
4784         struct adapter *sc = arg1;
4785         struct sbuf *sb;
4786         int rc;
4787         u64 nrate[NCHAN], orate[NCHAN];
4788
4789         rc = sysctl_wire_old_buffer(req, 0);
4790         if (rc != 0)
4791                 return (rc);
4792
4793         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4794         if (sb == NULL)
4795                 return (ENOMEM);
4796
4797         t4_get_chan_txrate(sc, nrate, orate);
4798         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4799                  "channel 3\n");
4800         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4801             nrate[0], nrate[1], nrate[2], nrate[3]);
4802         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4803             orate[0], orate[1], orate[2], orate[3]);
4804
4805         rc = sbuf_finish(sb);
4806         sbuf_delete(sb);
4807
4808         return (rc);
4809 }
4810 #endif
4811
4812 static inline void
4813 txq_start(struct ifnet *ifp, struct sge_txq *txq)
4814 {
4815         struct buf_ring *br;
4816         struct mbuf *m;
4817
4818         TXQ_LOCK_ASSERT_OWNED(txq);
4819
4820         br = txq->br;
4821         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4822         if (m)
4823                 t4_eth_tx(ifp, txq, m);
4824 }
4825
4826 void
4827 t4_tx_callout(void *arg)
4828 {
4829         struct sge_eq *eq = arg;
4830         struct adapter *sc;
4831
4832         if (EQ_TRYLOCK(eq) == 0)
4833                 goto reschedule;
4834
4835         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4836                 EQ_UNLOCK(eq);
4837 reschedule:
4838                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
4839                         callout_schedule(&eq->tx_callout, 1);
4840                 return;
4841         }
4842
4843         EQ_LOCK_ASSERT_OWNED(eq);
4844
4845         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4846
4847                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4848                         struct sge_txq *txq = arg;
4849                         struct port_info *pi = txq->ifp->if_softc;
4850
4851                         sc = pi->adapter;
4852                 } else {
4853                         struct sge_wrq *wrq = arg;
4854
4855                         sc = wrq->adapter;
4856                 }
4857
4858                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4859         }
4860
4861         EQ_UNLOCK(eq);
4862 }
4863
4864 void
4865 t4_tx_task(void *arg, int count)
4866 {
4867         struct sge_eq *eq = arg;
4868
4869         EQ_LOCK(eq);
4870         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4871                 struct sge_txq *txq = arg;
4872                 txq_start(txq->ifp, txq);
4873         } else {
4874                 struct sge_wrq *wrq = arg;
4875                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4876         }
4877         EQ_UNLOCK(eq);
4878 }
4879
4880 static uint32_t
4881 fconf_to_mode(uint32_t fconf)
4882 {
4883         uint32_t mode;
4884
4885         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4886             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4887
4888         if (fconf & F_FRAGMENTATION)
4889                 mode |= T4_FILTER_IP_FRAGMENT;
4890
4891         if (fconf & F_MPSHITTYPE)
4892                 mode |= T4_FILTER_MPS_HIT_TYPE;
4893
4894         if (fconf & F_MACMATCH)
4895                 mode |= T4_FILTER_MAC_IDX;
4896
4897         if (fconf & F_ETHERTYPE)
4898                 mode |= T4_FILTER_ETH_TYPE;
4899
4900         if (fconf & F_PROTOCOL)
4901                 mode |= T4_FILTER_IP_PROTO;
4902
4903         if (fconf & F_TOS)
4904                 mode |= T4_FILTER_IP_TOS;
4905
4906         if (fconf & F_VLAN)
4907                 mode |= T4_FILTER_VLAN;
4908
4909         if (fconf & F_VNIC_ID)
4910                 mode |= T4_FILTER_VNIC;
4911
4912         if (fconf & F_PORT)
4913                 mode |= T4_FILTER_PORT;
4914
4915         if (fconf & F_FCOE)
4916                 mode |= T4_FILTER_FCoE;
4917
4918         return (mode);
4919 }
4920
4921 static uint32_t
4922 mode_to_fconf(uint32_t mode)
4923 {
4924         uint32_t fconf = 0;
4925
4926         if (mode & T4_FILTER_IP_FRAGMENT)
4927                 fconf |= F_FRAGMENTATION;
4928
4929         if (mode & T4_FILTER_MPS_HIT_TYPE)
4930                 fconf |= F_MPSHITTYPE;
4931
4932         if (mode & T4_FILTER_MAC_IDX)
4933                 fconf |= F_MACMATCH;
4934
4935         if (mode & T4_FILTER_ETH_TYPE)
4936                 fconf |= F_ETHERTYPE;
4937
4938         if (mode & T4_FILTER_IP_PROTO)
4939                 fconf |= F_PROTOCOL;
4940
4941         if (mode & T4_FILTER_IP_TOS)
4942                 fconf |= F_TOS;
4943
4944         if (mode & T4_FILTER_VLAN)
4945                 fconf |= F_VLAN;
4946
4947         if (mode & T4_FILTER_VNIC)
4948                 fconf |= F_VNIC_ID;
4949
4950         if (mode & T4_FILTER_PORT)
4951                 fconf |= F_PORT;
4952
4953         if (mode & T4_FILTER_FCoE)
4954                 fconf |= F_FCOE;
4955
4956         return (fconf);
4957 }
4958
4959 static uint32_t
4960 fspec_to_fconf(struct t4_filter_specification *fs)
4961 {
4962         uint32_t fconf = 0;
4963
4964         if (fs->val.frag || fs->mask.frag)
4965                 fconf |= F_FRAGMENTATION;
4966
4967         if (fs->val.matchtype || fs->mask.matchtype)
4968                 fconf |= F_MPSHITTYPE;
4969
4970         if (fs->val.macidx || fs->mask.macidx)
4971                 fconf |= F_MACMATCH;
4972
4973         if (fs->val.ethtype || fs->mask.ethtype)
4974                 fconf |= F_ETHERTYPE;
4975
4976         if (fs->val.proto || fs->mask.proto)
4977                 fconf |= F_PROTOCOL;
4978
4979         if (fs->val.tos || fs->mask.tos)
4980                 fconf |= F_TOS;
4981
4982         if (fs->val.vlan_vld || fs->mask.vlan_vld)
4983                 fconf |= F_VLAN;
4984
4985         if (fs->val.vnic_vld || fs->mask.vnic_vld)
4986                 fconf |= F_VNIC_ID;
4987
4988         if (fs->val.iport || fs->mask.iport)
4989                 fconf |= F_PORT;
4990
4991         if (fs->val.fcoe || fs->mask.fcoe)
4992                 fconf |= F_FCOE;
4993
4994         return (fconf);
4995 }
4996
4997 static int
4998 get_filter_mode(struct adapter *sc, uint32_t *mode)
4999 {
5000         int rc;
5001         uint32_t fconf;
5002
5003         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
5004             "t4getfm");
5005         if (rc)
5006                 return (rc);
5007
5008         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
5009             A_TP_VLAN_PRI_MAP);
5010
5011         if (sc->filter_mode != fconf) {
5012                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
5013                     device_get_nameunit(sc->dev), sc->filter_mode, fconf);
5014                 sc->filter_mode = fconf;
5015         }
5016
5017         *mode = fconf_to_mode(sc->filter_mode);
5018
5019         end_synchronized_op(sc, LOCK_HELD);
5020         return (0);
5021 }
5022
5023 static int
5024 set_filter_mode(struct adapter *sc, uint32_t mode)
5025 {
5026         uint32_t fconf;
5027         int rc;
5028
5029         fconf = mode_to_fconf(mode);
5030
5031         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
5032             "t4setfm");
5033         if (rc)
5034                 return (rc);
5035
5036         if (sc->tids.ftids_in_use > 0) {
5037                 rc = EBUSY;
5038                 goto done;
5039         }
5040
5041 #ifdef TCP_OFFLOAD
5042         if (sc->offload_map) {
5043                 rc = EBUSY;
5044                 goto done;
5045         }
5046 #endif
5047
5048 #ifdef notyet
5049         rc = -t4_set_filter_mode(sc, fconf);
5050         if (rc == 0)
5051                 sc->filter_mode = fconf;
5052 #else
5053         rc = ENOTSUP;
5054 #endif
5055
5056 done:
5057         end_synchronized_op(sc, LOCK_HELD);
5058         return (rc);
5059 }
5060
5061 static inline uint64_t
5062 get_filter_hits(struct adapter *sc, uint32_t fid)
5063 {
5064         uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5065         uint64_t hits;
5066
5067         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
5068             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
5069         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
5070         hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
5071
5072         return (be64toh(hits));
5073 }
5074
5075 static int
5076 get_filter(struct adapter *sc, struct t4_filter *t)
5077 {
5078         int i, rc, nfilters = sc->tids.nftids;
5079         struct filter_entry *f;
5080
5081         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
5082             "t4getf");
5083         if (rc)
5084                 return (rc);
5085
5086         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
5087             t->idx >= nfilters) {
5088                 t->idx = 0xffffffff;
5089                 goto done;
5090         }
5091
5092         f = &sc->tids.ftid_tab[t->idx];
5093         for (i = t->idx; i < nfilters; i++, f++) {
5094                 if (f->valid) {
5095                         t->idx = i;
5096                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
5097                         t->smtidx = f->smtidx;
5098                         if (f->fs.hitcnts)
5099                                 t->hits = get_filter_hits(sc, t->idx);
5100                         else
5101                                 t->hits = UINT64_MAX;
5102                         t->fs = f->fs;
5103
5104                         goto done;
5105                 }
5106         }
5107
5108         t->idx = 0xffffffff;
5109 done:
5110         end_synchronized_op(sc, LOCK_HELD);
5111         return (0);
5112 }
5113
5114 static int
5115 set_filter(struct adapter *sc, struct t4_filter *t)
5116 {
5117         unsigned int nfilters, nports;
5118         struct filter_entry *f;
5119         int i, rc;
5120
5121         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
5122         if (rc)
5123                 return (rc);
5124
5125         nfilters = sc->tids.nftids;
5126         nports = sc->params.nports;
5127
5128         if (nfilters == 0) {
5129                 rc = ENOTSUP;
5130                 goto done;
5131         }
5132
5133         if (!(sc->flags & FULL_INIT_DONE)) {
5134                 rc = EAGAIN;
5135                 goto done;
5136         }
5137
5138         if (t->idx >= nfilters) {
5139                 rc = EINVAL;
5140                 goto done;
5141         }
5142
5143         /* Validate against the global filter mode */
5144         if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) {
5145                 rc = E2BIG;
5146                 goto done;
5147         }
5148
5149         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
5150                 rc = EINVAL;
5151                 goto done;
5152         }
5153
5154         if (t->fs.val.iport >= nports) {
5155                 rc = EINVAL;
5156                 goto done;
5157         }
5158
5159         /* Can't specify an iq if not steering to it */
5160         if (!t->fs.dirsteer && t->fs.iq) {
5161                 rc = EINVAL;
5162                 goto done;
5163         }
5164
5165         /* IPv6 filter idx must be 4 aligned */
5166         if (t->fs.type == 1 &&
5167             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
5168                 rc = EINVAL;
5169                 goto done;
5170         }
5171
5172         if (sc->tids.ftid_tab == NULL) {
5173                 KASSERT(sc->tids.ftids_in_use == 0,
5174                     ("%s: no memory allocated but filters_in_use > 0",
5175                     __func__));
5176
5177                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
5178                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
5179                 if (sc->tids.ftid_tab == NULL) {
5180                         rc = ENOMEM;
5181                         goto done;
5182                 }
5183                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
5184         }
5185
5186         for (i = 0; i < 4; i++) {
5187                 f = &sc->tids.ftid_tab[t->idx + i];
5188
5189                 if (f->pending || f->valid) {
5190                         rc = EBUSY;
5191                         goto done;
5192                 }
5193                 if (f->locked) {
5194                         rc = EPERM;
5195                         goto done;
5196                 }
5197
5198                 if (t->fs.type == 0)
5199                         break;
5200         }
5201
5202         f = &sc->tids.ftid_tab[t->idx];
5203         f->fs = t->fs;
5204
5205         rc = set_filter_wr(sc, t->idx);
5206 done:
5207         end_synchronized_op(sc, 0);
5208
5209         if (rc == 0) {
5210                 mtx_lock(&sc->tids.ftid_lock);
5211                 for (;;) {
5212                         if (f->pending == 0) {
5213                                 rc = f->valid ? 0 : EIO;
5214                                 break;
5215                         }
5216
5217                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
5218                             PCATCH, "t4setfw", 0)) {
5219                                 rc = EINPROGRESS;
5220                                 break;
5221                         }
5222                 }
5223                 mtx_unlock(&sc->tids.ftid_lock);
5224         }
5225         return (rc);
5226 }
5227
5228 static int
5229 del_filter(struct adapter *sc, struct t4_filter *t)
5230 {
5231         unsigned int nfilters;
5232         struct filter_entry *f;
5233         int rc;
5234
5235         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
5236         if (rc)
5237                 return (rc);
5238
5239         nfilters = sc->tids.nftids;
5240
5241         if (nfilters == 0) {
5242                 rc = ENOTSUP;
5243                 goto done;
5244         }
5245
5246         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
5247             t->idx >= nfilters) {
5248                 rc = EINVAL;
5249                 goto done;
5250         }
5251
5252         if (!(sc->flags & FULL_INIT_DONE)) {
5253                 rc = EAGAIN;
5254                 goto done;
5255         }
5256
5257         f = &sc->tids.ftid_tab[t->idx];
5258
5259         if (f->pending) {
5260                 rc = EBUSY;
5261                 goto done;
5262         }
5263         if (f->locked) {
5264                 rc = EPERM;
5265                 goto done;
5266         }
5267
5268         if (f->valid) {
5269                 t->fs = f->fs;  /* extra info for the caller */
5270                 rc = del_filter_wr(sc, t->idx);
5271         }
5272
5273 done:
5274         end_synchronized_op(sc, 0);
5275
5276         if (rc == 0) {
5277                 mtx_lock(&sc->tids.ftid_lock);
5278                 for (;;) {
5279                         if (f->pending == 0) {
5280                                 rc = f->valid ? EIO : 0;
5281                                 break;
5282                         }
5283
5284                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
5285                             PCATCH, "t4delfw", 0)) {
5286                                 rc = EINPROGRESS;
5287                                 break;
5288                         }
5289                 }
5290                 mtx_unlock(&sc->tids.ftid_lock);
5291         }
5292
5293         return (rc);
5294 }
5295
5296 static void
5297 clear_filter(struct filter_entry *f)
5298 {
5299         if (f->l2t)
5300                 t4_l2t_release(f->l2t);
5301
5302         bzero(f, sizeof (*f));
5303 }
5304
5305 static int
5306 set_filter_wr(struct adapter *sc, int fidx)
5307 {
5308         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
5309         struct wrqe *wr;
5310         struct fw_filter_wr *fwr;
5311         unsigned int ftid;
5312
5313         ASSERT_SYNCHRONIZED_OP(sc);
5314
5315         if (f->fs.newdmac || f->fs.newvlan) {
5316                 /* This filter needs an L2T entry; allocate one. */
5317                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
5318                 if (f->l2t == NULL)
5319                         return (EAGAIN);
5320                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
5321                     f->fs.dmac)) {
5322                         t4_l2t_release(f->l2t);
5323                         f->l2t = NULL;
5324                         return (ENOMEM);
5325                 }
5326         }
5327
5328         ftid = sc->tids.ftid_base + fidx;
5329
5330         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5331         if (wr == NULL)
5332                 return (ENOMEM);
5333
5334         fwr = wrtod(wr);
5335         bzero(fwr, sizeof (*fwr));
5336
5337         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
5338         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
5339         fwr->tid_to_iq =
5340             htobe32(V_FW_FILTER_WR_TID(ftid) |
5341                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
5342                 V_FW_FILTER_WR_NOREPLY(0) |
5343                 V_FW_FILTER_WR_IQ(f->fs.iq));
5344         fwr->del_filter_to_l2tix =
5345             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
5346                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
5347                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
5348                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
5349                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
5350                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
5351                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
5352                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
5353                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
5354                     f->fs.newvlan == VLAN_REWRITE) |
5355                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
5356                     f->fs.newvlan == VLAN_REWRITE) |
5357                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
5358                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
5359                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
5360                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
5361         fwr->ethtype = htobe16(f->fs.val.ethtype);
5362         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
5363         fwr->frag_to_ovlan_vldm =
5364             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
5365                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
5366                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
5367                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
5368                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
5369                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
5370         fwr->smac_sel = 0;
5371         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
5372             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
5373         fwr->maci_to_matchtypem =
5374             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
5375                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
5376                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
5377                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
5378                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
5379                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
5380                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
5381                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
5382         fwr->ptcl = f->fs.val.proto;
5383         fwr->ptclm = f->fs.mask.proto;
5384         fwr->ttyp = f->fs.val.tos;
5385         fwr->ttypm = f->fs.mask.tos;
5386         fwr->ivlan = htobe16(f->fs.val.vlan);
5387         fwr->ivlanm = htobe16(f->fs.mask.vlan);
5388         fwr->ovlan = htobe16(f->fs.val.vnic);
5389         fwr->ovlanm = htobe16(f->fs.mask.vnic);
5390         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
5391         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
5392         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
5393         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
5394         fwr->lp = htobe16(f->fs.val.dport);
5395         fwr->lpm = htobe16(f->fs.mask.dport);
5396         fwr->fp = htobe16(f->fs.val.sport);
5397         fwr->fpm = htobe16(f->fs.mask.sport);
5398         if (f->fs.newsmac)
5399                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
5400
5401         f->pending = 1;
5402         sc->tids.ftids_in_use++;
5403
5404         t4_wrq_tx(sc, wr);
5405         return (0);
5406 }
5407
5408 static int
5409 del_filter_wr(struct adapter *sc, int fidx)
5410 {
5411         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
5412         struct wrqe *wr;
5413         struct fw_filter_wr *fwr;
5414         unsigned int ftid;
5415
5416         ftid = sc->tids.ftid_base + fidx;
5417
5418         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5419         if (wr == NULL)
5420                 return (ENOMEM);
5421         fwr = wrtod(wr);
5422         bzero(fwr, sizeof (*fwr));
5423
5424         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
5425
5426         f->pending = 1;
5427         t4_wrq_tx(sc, wr);
5428         return (0);
5429 }
5430
5431 int
5432 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5433 {
5434         struct adapter *sc = iq->adapter;
5435         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
5436         unsigned int idx = GET_TID(rpl);
5437
5438         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5439             rss->opcode));
5440
5441         if (idx >= sc->tids.ftid_base &&
5442             (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
5443                 unsigned int rc = G_COOKIE(rpl->cookie);
5444                 struct filter_entry *f = &sc->tids.ftid_tab[idx];
5445
5446                 mtx_lock(&sc->tids.ftid_lock);
5447                 if (rc == FW_FILTER_WR_FLT_ADDED) {
5448                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
5449                             __func__, idx));
5450                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
5451                         f->pending = 0;  /* asynchronous setup completed */
5452                         f->valid = 1;
5453                 } else {
5454                         if (rc != FW_FILTER_WR_FLT_DELETED) {
5455                                 /* Add or delete failed, display an error */
5456                                 log(LOG_ERR,
5457                                     "filter %u setup failed with error %u\n",
5458                                     idx, rc);
5459                         }
5460
5461                         clear_filter(f);
5462                         sc->tids.ftids_in_use--;
5463                 }
5464                 wakeup(&sc->tids.ftid_tab);
5465                 mtx_unlock(&sc->tids.ftid_lock);
5466         }
5467
5468         return (0);
5469 }
5470
5471 static int
5472 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
5473 {
5474         int rc;
5475
5476         if (cntxt->cid > M_CTXTQID)
5477                 return (EINVAL);
5478
5479         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
5480             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
5481                 return (EINVAL);
5482
5483         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
5484         if (rc)
5485                 return (rc);
5486
5487         if (sc->flags & FW_OK) {
5488                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
5489                     &cntxt->data[0]);
5490                 if (rc == 0)
5491                         goto done;
5492         }
5493
5494         /*
5495          * Read via firmware failed or wasn't even attempted.  Read directly via
5496          * the backdoor.
5497          */
5498         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
5499 done:
5500         end_synchronized_op(sc, 0);
5501         return (rc);
5502 }
5503
5504 static int
5505 load_fw(struct adapter *sc, struct t4_data *fw)
5506 {
5507         int rc;
5508         uint8_t *fw_data;
5509
5510         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
5511         if (rc)
5512                 return (rc);
5513
5514         if (sc->flags & FULL_INIT_DONE) {
5515                 rc = EBUSY;
5516                 goto done;
5517         }
5518
5519         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
5520         if (fw_data == NULL) {
5521                 rc = ENOMEM;
5522                 goto done;
5523         }
5524
5525         rc = copyin(fw->data, fw_data, fw->len);
5526         if (rc == 0)
5527                 rc = -t4_load_fw(sc, fw_data, fw->len);
5528
5529         free(fw_data, M_CXGBE);
5530 done:
5531         end_synchronized_op(sc, 0);
5532         return (rc);
5533 }
5534
5535 static int
5536 read_card_mem(struct adapter *sc, struct t4_mem_range *mr)
5537 {
5538         uint32_t base, size, lo, hi, win, off, remaining, i, n;
5539         uint32_t *buf, *b;
5540         int rc;
5541
5542         /* reads are in multiples of 32 bits */
5543         if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
5544                 return (EINVAL);
5545
5546         /*
5547          * We don't want to deal with potential holes so we mandate that the
5548          * requested region must lie entirely within one of the 3 memories.
5549          */
5550         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5551         if (lo & F_EDRAM0_ENABLE) {
5552                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5553                 base = G_EDRAM0_BASE(hi) << 20;
5554                 size = G_EDRAM0_SIZE(hi) << 20;
5555                 if (size > 0 &&
5556                     mr->addr >= base && mr->addr < base + size &&
5557                     mr->addr + mr->len <= base + size)
5558                         goto proceed;
5559         }
5560         if (lo & F_EDRAM1_ENABLE) {
5561                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5562                 base = G_EDRAM1_BASE(hi) << 20;
5563                 size = G_EDRAM1_SIZE(hi) << 20;
5564                 if (size > 0 &&
5565                     mr->addr >= base && mr->addr < base + size &&
5566                     mr->addr + mr->len <= base + size)
5567                         goto proceed;
5568         }
5569         if (lo & F_EXT_MEM_ENABLE) {
5570                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5571                 base = G_EXT_MEM_BASE(hi) << 20;
5572                 size = G_EXT_MEM_SIZE(hi) << 20;
5573                 if (size > 0 &&
5574                     mr->addr >= base && mr->addr < base + size &&
5575                     mr->addr + mr->len <= base + size)
5576                         goto proceed;
5577         }
5578         return (ENXIO);
5579
5580 proceed:
5581         buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5582
5583         /*
5584          * Position the PCIe window (we use memwin2) to the 16B aligned area
5585          * just at/before the requested region.
5586          */
5587         win = mr->addr & ~0xf;
5588         off = mr->addr - win;  /* offset of the requested region in the win */
5589         remaining = mr->len;
5590
5591         while (remaining) {
5592                 t4_write_reg(sc,
5593                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5594                 t4_read_reg(sc,
5595                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5596
5597                 /* number of bytes that we'll copy in the inner loop */
5598                 n = min(remaining, MEMWIN2_APERTURE - off);
5599
5600                 for (i = 0; i < n; i += 4, remaining -= 4)
5601                         *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5602
5603                 win += MEMWIN2_APERTURE;
5604                 off = 0;
5605         }
5606
5607         rc = copyout(buf, mr->data, mr->len);
5608         free(buf, M_CXGBE);
5609
5610         return (rc);
5611 }
5612
5613 static int
5614 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
5615 {
5616         int rc;
5617
5618         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
5619                 return (EINVAL);
5620
5621         if (i2cd->len > 1) {
5622                 /* XXX: need fw support for longer reads in one go */
5623                 return (ENOTSUP);
5624         }
5625
5626         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
5627         if (rc)
5628                 return (rc);
5629         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
5630             i2cd->offset, &i2cd->data[0]);
5631         end_synchronized_op(sc, 0);
5632
5633         return (rc);
5634 }
5635
5636 int
5637 t4_os_find_pci_capability(struct adapter *sc, int cap)
5638 {
5639         int i;
5640
5641         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5642 }
5643
5644 int
5645 t4_os_pci_save_state(struct adapter *sc)
5646 {
5647         device_t dev;
5648         struct pci_devinfo *dinfo;
5649
5650         dev = sc->dev;
5651         dinfo = device_get_ivars(dev);
5652
5653         pci_cfg_save(dev, dinfo, 0);
5654         return (0);
5655 }
5656
5657 int
5658 t4_os_pci_restore_state(struct adapter *sc)
5659 {
5660         device_t dev;
5661         struct pci_devinfo *dinfo;
5662
5663         dev = sc->dev;
5664         dinfo = device_get_ivars(dev);
5665
5666         pci_cfg_restore(dev, dinfo);
5667         return (0);
5668 }
5669
5670 void
5671 t4_os_portmod_changed(const struct adapter *sc, int idx)
5672 {
5673         struct port_info *pi = sc->port[idx];
5674         static const char *mod_str[] = {
5675                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5676         };
5677
5678         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5679                 if_printf(pi->ifp, "transceiver unplugged.\n");
5680         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5681                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
5682         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5683                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5684         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
5685                 if_printf(pi->ifp, "%s transceiver inserted.\n",
5686                     mod_str[pi->mod_type]);
5687         } else {
5688                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5689                     pi->mod_type);
5690         }
5691 }
5692
5693 void
5694 t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5695 {
5696         struct port_info *pi = sc->port[idx];
5697         struct ifnet *ifp = pi->ifp;
5698
5699         if (link_stat) {
5700                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5701                 if_link_state_change(ifp, LINK_STATE_UP);
5702         } else
5703                 if_link_state_change(ifp, LINK_STATE_DOWN);
5704 }
5705
5706 void
5707 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
5708 {
5709         struct adapter *sc;
5710
5711         mtx_lock(&t4_list_lock);
5712         SLIST_FOREACH(sc, &t4_list, link) {
5713                 /*
5714                  * func should not make any assumptions about what state sc is
5715                  * in - the only guarantee is that sc->sc_lock is a valid lock.
5716                  */
5717                 func(sc, arg);
5718         }
5719         mtx_unlock(&t4_list_lock);
5720 }
5721
5722 static int
5723 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
5724 {
5725        return (0);
5726 }
5727
5728 static int
5729 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
5730 {
5731        return (0);
5732 }
5733
5734 static int
5735 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5736     struct thread *td)
5737 {
5738         int rc;
5739         struct adapter *sc = dev->si_drv1;
5740
5741         rc = priv_check(td, PRIV_DRIVER);
5742         if (rc != 0)
5743                 return (rc);
5744
5745         switch (cmd) {
5746         case CHELSIO_T4_GETREG: {
5747                 struct t4_reg *edata = (struct t4_reg *)data;
5748
5749                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5750                         return (EFAULT);
5751
5752                 if (edata->size == 4)
5753                         edata->val = t4_read_reg(sc, edata->addr);
5754                 else if (edata->size == 8)
5755                         edata->val = t4_read_reg64(sc, edata->addr);
5756                 else
5757                         return (EINVAL);
5758
5759                 break;
5760         }
5761         case CHELSIO_T4_SETREG: {
5762                 struct t4_reg *edata = (struct t4_reg *)data;
5763
5764                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5765                         return (EFAULT);
5766
5767                 if (edata->size == 4) {
5768                         if (edata->val & 0xffffffff00000000)
5769                                 return (EINVAL);
5770                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5771                 } else if (edata->size == 8)
5772                         t4_write_reg64(sc, edata->addr, edata->val);
5773                 else
5774                         return (EINVAL);
5775                 break;
5776         }
5777         case CHELSIO_T4_REGDUMP: {
5778                 struct t4_regdump *regs = (struct t4_regdump *)data;
5779                 int reglen = T4_REGDUMP_SIZE;
5780                 uint8_t *buf;
5781
5782                 if (regs->len < reglen) {
5783                         regs->len = reglen; /* hint to the caller */
5784                         return (ENOBUFS);
5785                 }
5786
5787                 regs->len = reglen;
5788                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5789                 t4_get_regs(sc, regs, buf);
5790                 rc = copyout(buf, regs->data, reglen);
5791                 free(buf, M_CXGBE);
5792                 break;
5793         }
5794         case CHELSIO_T4_GET_FILTER_MODE:
5795                 rc = get_filter_mode(sc, (uint32_t *)data);
5796                 break;
5797         case CHELSIO_T4_SET_FILTER_MODE:
5798                 rc = set_filter_mode(sc, *(uint32_t *)data);
5799                 break;
5800         case CHELSIO_T4_GET_FILTER:
5801                 rc = get_filter(sc, (struct t4_filter *)data);
5802                 break;
5803         case CHELSIO_T4_SET_FILTER:
5804                 rc = set_filter(sc, (struct t4_filter *)data);
5805                 break;
5806         case CHELSIO_T4_DEL_FILTER:
5807                 rc = del_filter(sc, (struct t4_filter *)data);
5808                 break;
5809         case CHELSIO_T4_GET_SGE_CONTEXT:
5810                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
5811                 break;
5812         case CHELSIO_T4_LOAD_FW:
5813                 rc = load_fw(sc, (struct t4_data *)data);
5814                 break;
5815         case CHELSIO_T4_GET_MEM:
5816                 rc = read_card_mem(sc, (struct t4_mem_range *)data);
5817                 break;
5818         case CHELSIO_T4_GET_I2C:
5819                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
5820                 break;
5821         case CHELSIO_T4_CLEAR_STATS: {
5822                 int i;
5823                 u_int port_id = *(uint32_t *)data;
5824                 struct port_info *pi;
5825
5826                 if (port_id >= sc->params.nports)
5827                         return (EINVAL);
5828
5829                 /* MAC stats */
5830                 t4_clr_port_stats(sc, port_id);
5831
5832                 pi = sc->port[port_id];
5833                 if (pi->flags & PORT_INIT_DONE) {
5834                         struct sge_rxq *rxq;
5835                         struct sge_txq *txq;
5836                         struct sge_wrq *wrq;
5837
5838                         for_each_rxq(pi, i, rxq) {
5839 #if defined(INET) || defined(INET6)
5840                                 rxq->lro.lro_queued = 0;
5841                                 rxq->lro.lro_flushed = 0;
5842 #endif
5843                                 rxq->rxcsum = 0;
5844                                 rxq->vlan_extraction = 0;
5845                         }
5846
5847                         for_each_txq(pi, i, txq) {
5848                                 txq->txcsum = 0;
5849                                 txq->tso_wrs = 0;
5850                                 txq->vlan_insertion = 0;
5851                                 txq->imm_wrs = 0;
5852                                 txq->sgl_wrs = 0;
5853                                 txq->txpkt_wrs = 0;
5854                                 txq->txpkts_wrs = 0;
5855                                 txq->txpkts_pkts = 0;
5856                                 txq->br->br_drops = 0;
5857                                 txq->no_dmamap = 0;
5858                                 txq->no_desc = 0;
5859                         }
5860
5861 #ifdef TCP_OFFLOAD
5862                         /* nothing to clear for each ofld_rxq */
5863
5864                         for_each_ofld_txq(pi, i, wrq) {
5865                                 wrq->tx_wrs = 0;
5866                                 wrq->no_desc = 0;
5867                         }
5868 #endif
5869                         wrq = &sc->sge.ctrlq[pi->port_id];
5870                         wrq->tx_wrs = 0;
5871                         wrq->no_desc = 0;
5872                 }
5873                 break;
5874         }
5875         default:
5876                 rc = EINVAL;
5877         }
5878
5879         return (rc);
5880 }
5881
5882 #ifdef TCP_OFFLOAD
5883 static int
5884 toe_capability(struct port_info *pi, int enable)
5885 {
5886         int rc;
5887         struct adapter *sc = pi->adapter;
5888
5889         ASSERT_SYNCHRONIZED_OP(sc);
5890
5891         if (!is_offload(sc))
5892                 return (ENODEV);
5893
5894         if (enable) {
5895                 if (!(sc->flags & FULL_INIT_DONE)) {
5896                         rc = cxgbe_init_synchronized(pi);
5897                         if (rc)
5898                                 return (rc);
5899                 }
5900
5901                 if (isset(&sc->offload_map, pi->port_id))
5902                         return (0);
5903
5904                 if (!(sc->flags & TOM_INIT_DONE)) {
5905                         rc = t4_activate_uld(sc, ULD_TOM);
5906                         if (rc == EAGAIN) {
5907                                 log(LOG_WARNING,
5908                                     "You must kldload t4_tom.ko before trying "
5909                                     "to enable TOE on a cxgbe interface.\n");
5910                         }
5911                         if (rc != 0)
5912                                 return (rc);
5913                         KASSERT(sc->tom_softc != NULL,
5914                             ("%s: TOM activated but softc NULL", __func__));
5915                         KASSERT(sc->flags & TOM_INIT_DONE,
5916                             ("%s: TOM activated but flag not set", __func__));
5917                 }
5918
5919                 setbit(&sc->offload_map, pi->port_id);
5920         } else {
5921                 if (!isset(&sc->offload_map, pi->port_id))
5922                         return (0);
5923
5924                 KASSERT(sc->flags & TOM_INIT_DONE,
5925                     ("%s: TOM never initialized?", __func__));
5926                 clrbit(&sc->offload_map, pi->port_id);
5927         }
5928
5929         return (0);
5930 }
5931
5932 /*
5933  * Add an upper layer driver to the global list.
5934  */
5935 int
5936 t4_register_uld(struct uld_info *ui)
5937 {
5938         int rc = 0;
5939         struct uld_info *u;
5940
5941         mtx_lock(&t4_uld_list_lock);
5942         SLIST_FOREACH(u, &t4_uld_list, link) {
5943             if (u->uld_id == ui->uld_id) {
5944                     rc = EEXIST;
5945                     goto done;
5946             }
5947         }
5948
5949         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5950         ui->refcount = 0;
5951 done:
5952         mtx_unlock(&t4_uld_list_lock);
5953         return (rc);
5954 }
5955
5956 int
5957 t4_unregister_uld(struct uld_info *ui)
5958 {
5959         int rc = EINVAL;
5960         struct uld_info *u;
5961
5962         mtx_lock(&t4_uld_list_lock);
5963
5964         SLIST_FOREACH(u, &t4_uld_list, link) {
5965             if (u == ui) {
5966                     if (ui->refcount > 0) {
5967                             rc = EBUSY;
5968                             goto done;
5969                     }
5970
5971                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5972                     rc = 0;
5973                     goto done;
5974             }
5975         }
5976 done:
5977         mtx_unlock(&t4_uld_list_lock);
5978         return (rc);
5979 }
5980
5981 int
5982 t4_activate_uld(struct adapter *sc, int id)
5983 {
5984         int rc = EAGAIN;
5985         struct uld_info *ui;
5986
5987         ASSERT_SYNCHRONIZED_OP(sc);
5988
5989         mtx_lock(&t4_uld_list_lock);
5990
5991         SLIST_FOREACH(ui, &t4_uld_list, link) {
5992                 if (ui->uld_id == id) {
5993                         rc = ui->activate(sc);
5994                         if (rc == 0)
5995                                 ui->refcount++;
5996                         goto done;
5997                 }
5998         }
5999 done:
6000         mtx_unlock(&t4_uld_list_lock);
6001
6002         return (rc);
6003 }
6004
6005 int
6006 t4_deactivate_uld(struct adapter *sc, int id)
6007 {
6008         int rc = EINVAL;
6009         struct uld_info *ui;
6010
6011         ASSERT_SYNCHRONIZED_OP(sc);
6012
6013         mtx_lock(&t4_uld_list_lock);
6014
6015         SLIST_FOREACH(ui, &t4_uld_list, link) {
6016                 if (ui->uld_id == id) {
6017                         rc = ui->deactivate(sc);
6018                         if (rc == 0)
6019                                 ui->refcount--;
6020                         goto done;
6021                 }
6022         }
6023 done:
6024         mtx_unlock(&t4_uld_list_lock);
6025
6026         return (rc);
6027 }
6028 #endif
6029
6030 /*
6031  * Come up with reasonable defaults for some of the tunables, provided they're
6032  * not set by the user (in which case we'll use the values as is).
6033  */
6034 static void
6035 tweak_tunables(void)
6036 {
6037         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
6038
6039         if (t4_ntxq10g < 1)
6040                 t4_ntxq10g = min(nc, NTXQ_10G);
6041
6042         if (t4_ntxq1g < 1)
6043                 t4_ntxq1g = min(nc, NTXQ_1G);
6044
6045         if (t4_nrxq10g < 1)
6046                 t4_nrxq10g = min(nc, NRXQ_10G);
6047
6048         if (t4_nrxq1g < 1)
6049                 t4_nrxq1g = min(nc, NRXQ_1G);
6050
6051 #ifdef TCP_OFFLOAD
6052         if (t4_nofldtxq10g < 1)
6053                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
6054
6055         if (t4_nofldtxq1g < 1)
6056                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
6057
6058         if (t4_nofldrxq10g < 1)
6059                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
6060
6061         if (t4_nofldrxq1g < 1)
6062                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
6063
6064         if (t4_toecaps_allowed == -1)
6065                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
6066 #else
6067         if (t4_toecaps_allowed == -1)
6068                 t4_toecaps_allowed = 0;
6069 #endif
6070
6071         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
6072                 t4_tmr_idx_10g = TMR_IDX_10G;
6073
6074         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
6075                 t4_pktc_idx_10g = PKTC_IDX_10G;
6076
6077         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
6078                 t4_tmr_idx_1g = TMR_IDX_1G;
6079
6080         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
6081                 t4_pktc_idx_1g = PKTC_IDX_1G;
6082
6083         if (t4_qsize_txq < 128)
6084                 t4_qsize_txq = 128;
6085
6086         if (t4_qsize_rxq < 128)
6087                 t4_qsize_rxq = 128;
6088         while (t4_qsize_rxq & 7)
6089                 t4_qsize_rxq++;
6090
6091         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
6092 }
6093
6094 static int
6095 t4_mod_event(module_t mod, int cmd, void *arg)
6096 {
6097         int rc = 0;
6098
6099         switch (cmd) {
6100         case MOD_LOAD:
6101                 t4_sge_modload();
6102                 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
6103                 SLIST_INIT(&t4_list);
6104 #ifdef TCP_OFFLOAD
6105                 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
6106                 SLIST_INIT(&t4_uld_list);
6107 #endif
6108                 tweak_tunables();
6109                 break;
6110
6111         case MOD_UNLOAD:
6112 #ifdef TCP_OFFLOAD
6113                 mtx_lock(&t4_uld_list_lock);
6114                 if (!SLIST_EMPTY(&t4_uld_list)) {
6115                         rc = EBUSY;
6116                         mtx_unlock(&t4_uld_list_lock);
6117                         break;
6118                 }
6119                 mtx_unlock(&t4_uld_list_lock);
6120                 mtx_destroy(&t4_uld_list_lock);
6121 #endif
6122                 mtx_lock(&t4_list_lock);
6123                 if (!SLIST_EMPTY(&t4_list)) {
6124                         rc = EBUSY;
6125                         mtx_unlock(&t4_list_lock);
6126                         break;
6127                 }
6128                 mtx_unlock(&t4_list_lock);
6129                 mtx_destroy(&t4_list_lock);
6130                 break;
6131         }
6132
6133         return (rc);
6134 }
6135
6136 static devclass_t t4_devclass;
6137 static devclass_t cxgbe_devclass;
6138
6139 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
6140 MODULE_VERSION(t4nex, 1);
6141
6142 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
6143 MODULE_VERSION(cxgbe, 1);