sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58
  59 #include "common/common.h"
  60 #include "common/t4_msg.h"
  61 #include "common/t4_regs.h"
  62 #include "common/t4_regs_values.h"
  63 #include "t4_ioctl.h"
  64 #include "t4_l2t.h"
  65
  66 /* T4 bus driver interface */
  67 static int t4_probe(device_t);
  68 static int t4_attach(device_t);
  69 static int t4_detach(device_t);
  70 static device_method_t t4_methods[] = {
  71         DEVMETHOD(device_probe,         t4_probe),
  72         DEVMETHOD(device_attach,        t4_attach),
  73         DEVMETHOD(device_detach,        t4_detach),
  74
  75         DEVMETHOD_END
  76 };
  77 static driver_t t4_driver = {
  78         "t4nex",
  79         t4_methods,
  80         sizeof(struct adapter)
  81 };
  82
  83
  84 /* T4 port (cxgbe) interface */
  85 static int cxgbe_probe(device_t);
  86 static int cxgbe_attach(device_t);
  87 static int cxgbe_detach(device_t);
  88 static device_method_t cxgbe_methods[] = {
  89         DEVMETHOD(device_probe,         cxgbe_probe),
  90         DEVMETHOD(device_attach,        cxgbe_attach),
  91         DEVMETHOD(device_detach,        cxgbe_detach),
  92         { 0, 0 }
  93 };
  94 static driver_t cxgbe_driver = {
  95         "cxgbe",
  96         cxgbe_methods,
  97         sizeof(struct port_info)
  98 };
  99
 100 static d_ioctl_t t4_ioctl;
 101 static d_open_t t4_open;
 102 static d_close_t t4_close;
 103
 104 static struct cdevsw t4_cdevsw = {
 105        .d_version = D_VERSION,
 106        .d_flags = 0,
 107        .d_open = t4_open,
 108        .d_close = t4_close,
 109        .d_ioctl = t4_ioctl,
 110        .d_name = "t4nex",
 111 };
 112
 113 /* ifnet + media interface */
 114 static void cxgbe_init(void *);
 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 117 static void cxgbe_qflush(struct ifnet *);
 118 static int cxgbe_media_change(struct ifnet *);
 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 120
 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
 122
 123 /*
 124  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 125  * then ADAPTER_LOCK, then t4_uld_list_lock.
 126  */
 127 static struct mtx t4_list_lock;
 128 static SLIST_HEAD(, adapter) t4_list;
 129 #ifdef TCP_OFFLOAD
 130 static struct mtx t4_uld_list_lock;
 131 static SLIST_HEAD(, uld_info) t4_uld_list;
 132 #endif
 133
 134 /*
 135  * Tunables.  See tweak_tunables() too.
 136  */
 137
 138 /*
 139  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 140  */
 141 #define NTXQ_10G 16
 142 static int t4_ntxq10g = -1;
 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 144
 145 #define NRXQ_10G 8
 146 static int t4_nrxq10g = -1;
 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 148
 149 #define NTXQ_1G 4
 150 static int t4_ntxq1g = -1;
 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 152
 153 #define NRXQ_1G 2
 154 static int t4_nrxq1g = -1;
 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 156
 157 #ifdef TCP_OFFLOAD
 158 #define NOFLDTXQ_10G 8
 159 static int t4_nofldtxq10g = -1;
 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 161
 162 #define NOFLDRXQ_10G 2
 163 static int t4_nofldrxq10g = -1;
 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 165
 166 #define NOFLDTXQ_1G 2
 167 static int t4_nofldtxq1g = -1;
 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 169
 170 #define NOFLDRXQ_1G 1
 171 static int t4_nofldrxq1g = -1;
 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 173 #endif
 174
 175 /*
 176  * Holdoff parameters for 10G and 1G ports.
 177  */
 178 #define TMR_IDX_10G 1
 179 static int t4_tmr_idx_10g = TMR_IDX_10G;
 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 181
 182 #define PKTC_IDX_10G (-1)
 183 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 185
 186 #define TMR_IDX_1G 1
 187 static int t4_tmr_idx_1g = TMR_IDX_1G;
 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 189
 190 #define PKTC_IDX_1G (-1)
 191 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 193
 194 /*
 195  * Size (# of entries) of each tx and rx queue.
 196  */
 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 199
 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 202
 203 /*
 204  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 205  */
 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 208
 209 /*
 210  * Configuration file.
 211  */
 212 static char t4_cfg_file[32] = "default";
 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 214
 215 /*
 216  * ASIC features that will be used.  Disable the ones you don't want so that the
 217  * chip resources aren't wasted on features that will not be used.
 218  */
 219 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 220 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 221
 222 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 223 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 224
 225 static int t4_toecaps_allowed = -1;
 226 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 227
 228 static int t4_rdmacaps_allowed = 0;
 229 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 230
 231 static int t4_iscsicaps_allowed = 0;
 232 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 233
 234 static int t4_fcoecaps_allowed = 0;
 235 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 236
 237 struct intrs_and_queues {
 238         int intr_type;          /* INTx, MSI, or MSI-X */
 239         int nirq;               /* Number of vectors */
 240         int intr_flags;
 241         int ntxq10g;            /* # of NIC txq's for each 10G port */
 242         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 243         int ntxq1g;             /* # of NIC txq's for each 1G port */
 244         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 245 #ifdef TCP_OFFLOAD
 246         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 247         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 248         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 249         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 250 #endif
 251 };
 252
 253 struct filter_entry {
 254         uint32_t valid:1;       /* filter allocated and valid */
 255         uint32_t locked:1;      /* filter is administratively locked */
 256         uint32_t pending:1;     /* filter action is pending firmware reply */
 257         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 258         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 259
 260         struct t4_filter_specification fs;
 261 };
 262
 263 enum {
 264         XGMAC_MTU       = (1 << 0),
 265         XGMAC_PROMISC   = (1 << 1),
 266         XGMAC_ALLMULTI  = (1 << 2),
 267         XGMAC_VLANEX    = (1 << 3),
 268         XGMAC_UCADDR    = (1 << 4),
 269         XGMAC_MCADDRS   = (1 << 5),
 270
 271         XGMAC_ALL       = 0xffff
 272 };
 273
 274 static int map_bars(struct adapter *);
 275 static void setup_memwin(struct adapter *);
 276 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 277     struct intrs_and_queues *);
 278 static int prep_firmware(struct adapter *);
 279 static int upload_config_file(struct adapter *, const struct firmware *,
 280     uint32_t *, uint32_t *);
 281 static int partition_resources(struct adapter *, const struct firmware *);
 282 static int get_params__pre_init(struct adapter *);
 283 static int get_params__post_init(struct adapter *);
 284 static void t4_set_desc(struct adapter *);
 285 static void build_medialist(struct port_info *);
 286 static int update_mac_settings(struct port_info *, int);
 287 static int cxgbe_init_locked(struct port_info *);
 288 static int cxgbe_init_synchronized(struct port_info *);
 289 static int cxgbe_uninit_locked(struct port_info *);
 290 static int cxgbe_uninit_synchronized(struct port_info *);
 291 static int setup_intr_handlers(struct adapter *);
 292 static int adapter_full_init(struct adapter *);
 293 static int adapter_full_uninit(struct adapter *);
 294 static int port_full_init(struct port_info *);
 295 static int port_full_uninit(struct port_info *);
 296 static void quiesce_eq(struct adapter *, struct sge_eq *);
 297 static void quiesce_iq(struct adapter *, struct sge_iq *);
 298 static void quiesce_fl(struct adapter *, struct sge_fl *);
 299 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 300     driver_intr_t *, void *, char *);
 301 static int t4_free_irq(struct adapter *, struct irq *);
 302 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 303     unsigned int);
 304 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 305 static void cxgbe_tick(void *);
 306 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 307 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 308     struct mbuf *);
 309 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 310 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 311 static int t4_sysctls(struct adapter *);
 312 static int cxgbe_sysctls(struct port_info *);
 313 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 314 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 315 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 316 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 317 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 318 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 319 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 320 #ifdef SBUF_DRAIN
 321 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 322 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 323 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 324 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 325 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 326 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 327 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 328 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 329 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 330 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 331 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 332 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 333 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 334 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 335 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 336 #endif
 337 static inline void txq_start(struct ifnet *, struct sge_txq *);
 338 static uint32_t fconf_to_mode(uint32_t);
 339 static uint32_t mode_to_fconf(uint32_t);
 340 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 341 static int get_filter_mode(struct adapter *, uint32_t *);
 342 static int set_filter_mode(struct adapter *, uint32_t);
 343 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 344 static int get_filter(struct adapter *, struct t4_filter *);
 345 static int set_filter(struct adapter *, struct t4_filter *);
 346 static int del_filter(struct adapter *, struct t4_filter *);
 347 static void clear_filter(struct filter_entry *);
 348 static int set_filter_wr(struct adapter *, int);
 349 static int del_filter_wr(struct adapter *, int);
 350 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 351 static int read_card_mem(struct adapter *, struct t4_mem_range *);
 352 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 353 #ifdef TCP_OFFLOAD
 354 static int toe_capability(struct port_info *, int);
 355 #endif
 356 static int t4_mod_event(module_t, int, void *);
 357
 358 struct t4_pciids {
 359         uint16_t device;
 360         char *desc;
 361 } t4_pciids[] = {
 362         {0xa000, "Chelsio Terminator 4 FPGA"},
 363         {0x4400, "Chelsio T440-dbg"},
 364         {0x4401, "Chelsio T420-CR"},
 365         {0x4402, "Chelsio T422-CR"},
 366         {0x4403, "Chelsio T440-CR"},
 367         {0x4404, "Chelsio T420-BCH"},
 368         {0x4405, "Chelsio T440-BCH"},
 369         {0x4406, "Chelsio T440-CH"},
 370         {0x4407, "Chelsio T420-SO"},
 371         {0x4408, "Chelsio T420-CX"},
 372         {0x4409, "Chelsio T420-BT"},
 373         {0x440a, "Chelsio T404-BT"},
 374 };
 375
 376 #ifdef TCP_OFFLOAD
 377 /*
 378  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 379  * exactly the same for both rxq and ofld_rxq.
 380  */
 381 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 382 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 383 #endif
 384
 385 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 386 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 387 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 388
 389 static int
 390 t4_probe(device_t dev)
 391 {
 392         int i;
 393         uint16_t v = pci_get_vendor(dev);
 394         uint16_t d = pci_get_device(dev);
 395         uint8_t f = pci_get_function(dev);
 396
 397         if (v != PCI_VENDOR_ID_CHELSIO)
 398                 return (ENXIO);
 399
 400         /* Attach only to PF0 of the FPGA */
 401         if (d == 0xa000 && f != 0)
 402                 return (ENXIO);
 403
 404         for (i = 0; i < nitems(t4_pciids); i++) {
 405                 if (d == t4_pciids[i].device) {
 406                         device_set_desc(dev, t4_pciids[i].desc);
 407                         return (BUS_PROBE_DEFAULT);
 408                 }
 409         }
 410
 411         return (ENXIO);
 412 }
 413
 414 static int
 415 t4_attach(device_t dev)
 416 {
 417         struct adapter *sc;
 418         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 419         struct intrs_and_queues iaq;
 420         struct sge *s;
 421 #ifdef TCP_OFFLOAD
 422         int ofld_rqidx, ofld_tqidx;
 423 #endif
 424
 425         sc = device_get_softc(dev);
 426         sc->dev = dev;
 427
 428         pci_enable_busmaster(dev);
 429         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 430                 uint32_t v;
 431
 432                 pci_set_max_read_req(dev, 4096);
 433                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 434                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 435                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 436         }
 437
 438         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 439             device_get_nameunit(dev));
 440         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 441         mtx_lock(&t4_list_lock);
 442         SLIST_INSERT_HEAD(&t4_list, sc, link);
 443         mtx_unlock(&t4_list_lock);
 444
 445         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 446         TAILQ_INIT(&sc->sfl);
 447         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 448
 449         rc = map_bars(sc);
 450         if (rc != 0)
 451                 goto done; /* error message displayed already */
 452
 453         /*
 454          * This is the real PF# to which we're attaching.  Works from within PCI
 455          * passthrough environments too, where pci_get_function() could return a
 456          * different PF# depending on the passthrough configuration.  We need to
 457          * use the real PF# in all our communication with the firmware.
 458          */
 459         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 460         sc->mbox = sc->pf;
 461
 462         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 463         sc->an_handler = an_not_handled;
 464         for (i = 0; i < nitems(sc->cpl_handler); i++)
 465                 sc->cpl_handler[i] = cpl_not_handled;
 466         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 467                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 468         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 469
 470         /* Prepare the adapter for operation */
 471         rc = -t4_prep_adapter(sc);
 472         if (rc != 0) {
 473                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 474                 goto done;
 475         }
 476
 477         /*
 478          * Do this really early, with the memory windows set up even before the
 479          * character device.  The userland tool's register i/o and mem read
 480          * will work even in "recovery mode".
 481          */
 482         setup_memwin(sc);
 483         sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
 484             GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
 485         sc->cdev->si_drv1 = sc;
 486
 487         /* Go no further if recovery mode has been requested. */
 488         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 489                 device_printf(dev, "recovery mode.\n");
 490                 goto done;
 491         }
 492
 493         /* Prepare the firmware for operation */
 494         rc = prep_firmware(sc);
 495         if (rc != 0)
 496                 goto done; /* error message displayed already */
 497
 498         rc = get_params__pre_init(sc);
 499         if (rc != 0)
 500                 goto done; /* error message displayed already */
 501
 502         rc = t4_sge_init(sc);
 503         if (rc != 0)
 504                 goto done; /* error message displayed already */
 505
 506         if (sc->flags & MASTER_PF) {
 507                 /* get basic stuff going */
 508                 rc = -t4_fw_initialize(sc, sc->mbox);
 509                 if (rc != 0) {
 510                         device_printf(dev, "early init failed: %d.\n", rc);
 511                         goto done;
 512                 }
 513         }
 514
 515         rc = get_params__post_init(sc);
 516         if (rc != 0)
 517                 goto done; /* error message displayed already */
 518
 519         if (sc->flags & MASTER_PF) {
 520                 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
 521
 522                 /* final tweaks to some settings */
 523
 524                 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
 525                     sc->params.b_wnd);
 526                 /* 4K, 16K, 64K, 256K DDP "page sizes" */
 527                 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) |
 528                     V_HPZ2(4) | V_HPZ3(6));
 529                 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB);
 530                 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 |
 531                     F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3,
 532                     F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 |
 533                     F_TUNNELCNGDROP3);
 534                 t4_set_reg_field(sc, A_TP_PARA_REG5,
 535                     V_INDICATESIZE(M_INDICATESIZE) |
 536                     F_REARMDDPOFFSET | F_RESETDDPOFFSET,
 537                     V_INDICATESIZE(indsz) |
 538                     F_REARMDDPOFFSET | F_RESETDDPOFFSET);
 539         } else {
 540                 /*
 541                  * XXX: Verify that we can live with whatever the master driver
 542                  * has done so far, and hope that it doesn't change any global
 543                  * setting from underneath us in the future.
 544                  */
 545         }
 546
 547         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
 548             A_TP_VLAN_PRI_MAP);
 549
 550         for (i = 0; i < NCHAN; i++)
 551                 sc->params.tp.tx_modq[i] = i;
 552
 553         rc = t4_create_dma_tag(sc);
 554         if (rc != 0)
 555                 goto done; /* error message displayed already */
 556
 557         /*
 558          * First pass over all the ports - allocate VIs and initialize some
 559          * basic parameters like mac address, port type, etc.  We also figure
 560          * out whether a port is 10G or 1G and use that information when
 561          * calculating how many interrupts to attempt to allocate.
 562          */
 563         n10g = n1g = 0;
 564         for_each_port(sc, i) {
 565                 struct port_info *pi;
 566
 567                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 568                 sc->port[i] = pi;
 569
 570                 /* These must be set before t4_port_init */
 571                 pi->adapter = sc;
 572                 pi->port_id = i;
 573
 574                 /* Allocate the vi and initialize parameters like mac addr */
 575                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 576                 if (rc != 0) {
 577                         device_printf(dev, "unable to initialize port %d: %d\n",
 578                             i, rc);
 579                         free(pi, M_CXGBE);
 580                         sc->port[i] = NULL;
 581                         goto done;
 582                 }
 583
 584                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 585                     device_get_nameunit(dev), i);
 586                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 587
 588                 if (is_10G_port(pi)) {
 589                         n10g++;
 590                         pi->tmr_idx = t4_tmr_idx_10g;
 591                         pi->pktc_idx = t4_pktc_idx_10g;
 592                 } else {
 593                         n1g++;
 594                         pi->tmr_idx = t4_tmr_idx_1g;
 595                         pi->pktc_idx = t4_pktc_idx_1g;
 596                 }
 597
 598                 pi->xact_addr_filt = -1;
 599
 600                 pi->qsize_rxq = t4_qsize_rxq;
 601                 pi->qsize_txq = t4_qsize_txq;
 602
 603                 pi->dev = device_add_child(dev, "cxgbe", -1);
 604                 if (pi->dev == NULL) {
 605                         device_printf(dev,
 606                             "failed to add device for port %d.\n", i);
 607                         rc = ENXIO;
 608                         goto done;
 609                 }
 610                 device_set_softc(pi->dev, pi);
 611         }
 612
 613         /*
 614          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 615          */
 616         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 617         if (rc != 0)
 618                 goto done; /* error message displayed already */
 619
 620         sc->intr_type = iaq.intr_type;
 621         sc->intr_count = iaq.nirq;
 622         sc->flags |= iaq.intr_flags;
 623
 624         s = &sc->sge;
 625         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 626         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 627         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 628         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 629         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 630
 631 #ifdef TCP_OFFLOAD
 632         if (is_offload(sc)) {
 633
 634                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 635                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 636                 s->neq += s->nofldtxq + s->nofldrxq;
 637                 s->niq += s->nofldrxq;
 638
 639                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 640                     M_CXGBE, M_ZERO | M_WAITOK);
 641                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 642                     M_CXGBE, M_ZERO | M_WAITOK);
 643         }
 644 #endif
 645
 646         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 647             M_ZERO | M_WAITOK);
 648         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 649             M_ZERO | M_WAITOK);
 650         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 651             M_ZERO | M_WAITOK);
 652         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 653             M_ZERO | M_WAITOK);
 654         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 655             M_ZERO | M_WAITOK);
 656
 657         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 658             M_ZERO | M_WAITOK);
 659
 660         t4_init_l2t(sc, M_WAITOK);
 661
 662         /*
 663          * Second pass over the ports.  This time we know the number of rx and
 664          * tx queues that each port should get.
 665          */
 666         rqidx = tqidx = 0;
 667 #ifdef TCP_OFFLOAD
 668         ofld_rqidx = ofld_tqidx = 0;
 669 #endif
 670         for_each_port(sc, i) {
 671                 struct port_info *pi = sc->port[i];
 672
 673                 if (pi == NULL)
 674                         continue;
 675
 676                 pi->first_rxq = rqidx;
 677                 pi->first_txq = tqidx;
 678                 if (is_10G_port(pi)) {
 679                         pi->nrxq = iaq.nrxq10g;
 680                         pi->ntxq = iaq.ntxq10g;
 681                 } else {
 682                         pi->nrxq = iaq.nrxq1g;
 683                         pi->ntxq = iaq.ntxq1g;
 684                 }
 685
 686                 rqidx += pi->nrxq;
 687                 tqidx += pi->ntxq;
 688
 689 #ifdef TCP_OFFLOAD
 690                 if (is_offload(sc)) {
 691                         pi->first_ofld_rxq = ofld_rqidx;
 692                         pi->first_ofld_txq = ofld_tqidx;
 693                         if (is_10G_port(pi)) {
 694                                 pi->nofldrxq = iaq.nofldrxq10g;
 695                                 pi->nofldtxq = iaq.nofldtxq10g;
 696                         } else {
 697                                 pi->nofldrxq = iaq.nofldrxq1g;
 698                                 pi->nofldtxq = iaq.nofldtxq1g;
 699                         }
 700                         ofld_rqidx += pi->nofldrxq;
 701                         ofld_tqidx += pi->nofldtxq;
 702                 }
 703 #endif
 704         }
 705
 706         rc = setup_intr_handlers(sc);
 707         if (rc != 0) {
 708                 device_printf(dev,
 709                     "failed to setup interrupt handlers: %d\n", rc);
 710                 goto done;
 711         }
 712
 713         rc = bus_generic_attach(dev);
 714         if (rc != 0) {
 715                 device_printf(dev,
 716                     "failed to attach all child ports: %d\n", rc);
 717                 goto done;
 718         }
 719
 720         device_printf(dev,
 721             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 722             sc->params.pci.width, sc->params.nports, sc->intr_count,
 723             sc->intr_type == INTR_MSIX ? "MSI-X" :
 724             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 725             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 726
 727         t4_set_desc(sc);
 728
 729 done:
 730         if (rc != 0 && sc->cdev) {
 731                 /* cdev was created and so cxgbetool works; recover that way. */
 732                 device_printf(dev,
 733                     "error during attach, adapter is now in recovery mode.\n");
 734                 rc = 0;
 735         }
 736
 737         if (rc != 0)
 738                 t4_detach(dev);
 739         else
 740                 t4_sysctls(sc);
 741
 742         return (rc);
 743 }
 744
 745 /*
 746  * Idempotent
 747  */
 748 static int
 749 t4_detach(device_t dev)
 750 {
 751         struct adapter *sc;
 752         struct port_info *pi;
 753         int i, rc;
 754
 755         sc = device_get_softc(dev);
 756
 757         if (sc->flags & FULL_INIT_DONE)
 758                 t4_intr_disable(sc);
 759
 760         if (sc->cdev) {
 761                 destroy_dev(sc->cdev);
 762                 sc->cdev = NULL;
 763         }
 764
 765         rc = bus_generic_detach(dev);
 766         if (rc) {
 767                 device_printf(dev,
 768                     "failed to detach child devices: %d\n", rc);
 769                 return (rc);
 770         }
 771
 772         for (i = 0; i < sc->intr_count; i++)
 773                 t4_free_irq(sc, &sc->irq[i]);
 774
 775         for (i = 0; i < MAX_NPORTS; i++) {
 776                 pi = sc->port[i];
 777                 if (pi) {
 778                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 779                         if (pi->dev)
 780                                 device_delete_child(dev, pi->dev);
 781
 782                         mtx_destroy(&pi->pi_lock);
 783                         free(pi, M_CXGBE);
 784                 }
 785         }
 786
 787         if (sc->flags & FULL_INIT_DONE)
 788                 adapter_full_uninit(sc);
 789
 790         if (sc->flags & FW_OK)
 791                 t4_fw_bye(sc, sc->mbox);
 792
 793         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 794                 pci_release_msi(dev);
 795
 796         if (sc->regs_res)
 797                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 798                     sc->regs_res);
 799
 800         if (sc->msix_res)
 801                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 802                     sc->msix_res);
 803
 804         if (sc->l2t)
 805                 t4_free_l2t(sc->l2t);
 806
 807 #ifdef TCP_OFFLOAD
 808         free(sc->sge.ofld_rxq, M_CXGBE);
 809         free(sc->sge.ofld_txq, M_CXGBE);
 810 #endif
 811         free(sc->irq, M_CXGBE);
 812         free(sc->sge.rxq, M_CXGBE);
 813         free(sc->sge.txq, M_CXGBE);
 814         free(sc->sge.ctrlq, M_CXGBE);
 815         free(sc->sge.iqmap, M_CXGBE);
 816         free(sc->sge.eqmap, M_CXGBE);
 817         free(sc->tids.ftid_tab, M_CXGBE);
 818         t4_destroy_dma_tag(sc);
 819         if (mtx_initialized(&sc->sc_lock)) {
 820                 mtx_lock(&t4_list_lock);
 821                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 822                 mtx_unlock(&t4_list_lock);
 823                 mtx_destroy(&sc->sc_lock);
 824         }
 825
 826         if (mtx_initialized(&sc->sfl_lock))
 827                 mtx_destroy(&sc->sfl_lock);
 828
 829         bzero(sc, sizeof(*sc));
 830
 831         return (0);
 832 }
 833
 834
 835 static int
 836 cxgbe_probe(device_t dev)
 837 {
 838         char buf[128];
 839         struct port_info *pi = device_get_softc(dev);
 840
 841         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 842         device_set_desc_copy(dev, buf);
 843
 844         return (BUS_PROBE_DEFAULT);
 845 }
 846
 847 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 848     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 849     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
 850 #define T4_CAP_ENABLE (T4_CAP)
 851
 852 static int
 853 cxgbe_attach(device_t dev)
 854 {
 855         struct port_info *pi = device_get_softc(dev);
 856         struct ifnet *ifp;
 857
 858         /* Allocate an ifnet and set it up */
 859         ifp = if_alloc(IFT_ETHER);
 860         if (ifp == NULL) {
 861                 device_printf(dev, "Cannot allocate ifnet\n");
 862                 return (ENOMEM);
 863         }
 864         pi->ifp = ifp;
 865         ifp->if_softc = pi;
 866
 867         callout_init(&pi->tick, CALLOUT_MPSAFE);
 868
 869         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 870         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 871
 872         ifp->if_init = cxgbe_init;
 873         ifp->if_ioctl = cxgbe_ioctl;
 874         ifp->if_transmit = cxgbe_transmit;
 875         ifp->if_qflush = cxgbe_qflush;
 876
 877         ifp->if_capabilities = T4_CAP;
 878 #ifdef TCP_OFFLOAD
 879         if (is_offload(pi->adapter))
 880                 ifp->if_capabilities |= IFCAP_TOE4;
 881 #endif
 882         ifp->if_capenable = T4_CAP_ENABLE;
 883         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 884             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 885
 886         /* Initialize ifmedia for this port */
 887         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 888             cxgbe_media_status);
 889         build_medialist(pi);
 890
 891         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 892             EVENTHANDLER_PRI_ANY);
 893
 894         ether_ifattach(ifp, pi->hw_addr);
 895
 896 #ifdef TCP_OFFLOAD
 897         if (is_offload(pi->adapter)) {
 898                 device_printf(dev,
 899                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
 900                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
 901         } else
 902 #endif
 903                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
 904
 905         cxgbe_sysctls(pi);
 906
 907         return (0);
 908 }
 909
 910 static int
 911 cxgbe_detach(device_t dev)
 912 {
 913         struct port_info *pi = device_get_softc(dev);
 914         struct adapter *sc = pi->adapter;
 915         struct ifnet *ifp = pi->ifp;
 916
 917         /* Tell if_ioctl and if_init that the port is going away */
 918         ADAPTER_LOCK(sc);
 919         SET_DOOMED(pi);
 920         wakeup(&sc->flags);
 921         while (IS_BUSY(sc))
 922                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 923         SET_BUSY(sc);
 924         ADAPTER_UNLOCK(sc);
 925
 926         if (pi->vlan_c)
 927                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
 928
 929         PORT_LOCK(pi);
 930         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 931         callout_stop(&pi->tick);
 932         PORT_UNLOCK(pi);
 933         callout_drain(&pi->tick);
 934
 935         /* Let detach proceed even if these fail. */
 936         cxgbe_uninit_synchronized(pi);
 937         port_full_uninit(pi);
 938
 939         ifmedia_removeall(&pi->media);
 940         ether_ifdetach(pi->ifp);
 941         if_free(pi->ifp);
 942
 943         ADAPTER_LOCK(sc);
 944         CLR_BUSY(sc);
 945         wakeup_one(&sc->flags);
 946         ADAPTER_UNLOCK(sc);
 947
 948         return (0);
 949 }
 950
 951 static void
 952 cxgbe_init(void *arg)
 953 {
 954         struct port_info *pi = arg;
 955         struct adapter *sc = pi->adapter;
 956
 957         ADAPTER_LOCK(sc);
 958         cxgbe_init_locked(pi); /* releases adapter lock */
 959         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 960 }
 961
 962 static int
 963 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 964 {
 965         int rc = 0, mtu, flags;
 966         struct port_info *pi = ifp->if_softc;
 967         struct adapter *sc = pi->adapter;
 968         struct ifreq *ifr = (struct ifreq *)data;
 969         uint32_t mask;
 970
 971         switch (cmd) {
 972         case SIOCSIFMTU:
 973                 ADAPTER_LOCK(sc);
 974                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
 975                 if (rc) {
 976 fail:
 977                         ADAPTER_UNLOCK(sc);
 978                         return (rc);
 979                 }
 980
 981                 mtu = ifr->ifr_mtu;
 982                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
 983                         rc = EINVAL;
 984                 } else {
 985                         ifp->if_mtu = mtu;
 986                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 987                                 t4_update_fl_bufsize(ifp);
 988                                 PORT_LOCK(pi);
 989                                 rc = update_mac_settings(pi, XGMAC_MTU);
 990                                 PORT_UNLOCK(pi);
 991                         }
 992                 }
 993                 ADAPTER_UNLOCK(sc);
 994                 break;
 995
 996         case SIOCSIFFLAGS:
 997                 ADAPTER_LOCK(sc);
 998                 if (IS_DOOMED(pi)) {
 999                         rc = ENXIO;
1000                         goto fail;
1001                 }
1002                 if (ifp->if_flags & IFF_UP) {
1003                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1004                                 flags = pi->if_flags;
1005                                 if ((ifp->if_flags ^ flags) &
1006                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1007                                         if (IS_BUSY(sc)) {
1008                                                 rc = EBUSY;
1009                                                 goto fail;
1010                                         }
1011                                         PORT_LOCK(pi);
1012                                         rc = update_mac_settings(pi,
1013                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1014                                         PORT_UNLOCK(pi);
1015                                 }
1016                                 ADAPTER_UNLOCK(sc);
1017                         } else
1018                                 rc = cxgbe_init_locked(pi);
1019                         pi->if_flags = ifp->if_flags;
1020                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1021                         rc = cxgbe_uninit_locked(pi);
1022                 else
1023                         ADAPTER_UNLOCK(sc);
1024
1025                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1026                 break;
1027
1028         case SIOCADDMULTI:
1029         case SIOCDELMULTI: /* these two can be called with a mutex held :-( */
1030                 ADAPTER_LOCK(sc);
1031                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1032                 if (rc)
1033                         goto fail;
1034
1035                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1036                         PORT_LOCK(pi);
1037                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1038                         PORT_UNLOCK(pi);
1039                 }
1040                 ADAPTER_UNLOCK(sc);
1041                 break;
1042
1043         case SIOCSIFCAP:
1044                 ADAPTER_LOCK(sc);
1045                 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1046                 if (rc)
1047                         goto fail;
1048
1049                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1050                 if (mask & IFCAP_TXCSUM) {
1051                         ifp->if_capenable ^= IFCAP_TXCSUM;
1052                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1053
1054                         if (IFCAP_TSO4 & ifp->if_capenable &&
1055                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1056                                 ifp->if_capenable &= ~IFCAP_TSO4;
1057                                 if_printf(ifp,
1058                                     "tso4 disabled due to -txcsum.\n");
1059                         }
1060                 }
1061                 if (mask & IFCAP_TXCSUM_IPV6) {
1062                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1063                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1064
1065                         if (IFCAP_TSO6 & ifp->if_capenable &&
1066                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1067                                 ifp->if_capenable &= ~IFCAP_TSO6;
1068                                 if_printf(ifp,
1069                                     "tso6 disabled due to -txcsum6.\n");
1070                         }
1071                 }
1072                 if (mask & IFCAP_RXCSUM)
1073                         ifp->if_capenable ^= IFCAP_RXCSUM;
1074                 if (mask & IFCAP_RXCSUM_IPV6)
1075                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1076
1077                 /*
1078                  * Note that we leave CSUM_TSO alone (it is always set).  The
1079                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1080                  * sending a TSO request our way, so it's sufficient to toggle
1081                  * IFCAP_TSOx only.
1082                  */
1083                 if (mask & IFCAP_TSO4) {
1084                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1085                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1086                                 if_printf(ifp, "enable txcsum first.\n");
1087                                 rc = EAGAIN;
1088                                 goto fail;
1089                         }
1090                         ifp->if_capenable ^= IFCAP_TSO4;
1091                 }
1092                 if (mask & IFCAP_TSO6) {
1093                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1094                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1095                                 if_printf(ifp, "enable txcsum6 first.\n");
1096                                 rc = EAGAIN;
1097                                 goto fail;
1098                         }
1099                         ifp->if_capenable ^= IFCAP_TSO6;
1100                 }
1101                 if (mask & IFCAP_LRO) {
1102 #if defined(INET) || defined(INET6)
1103                         int i;
1104                         struct sge_rxq *rxq;
1105
1106                         ifp->if_capenable ^= IFCAP_LRO;
1107                         for_each_rxq(pi, i, rxq) {
1108                                 if (ifp->if_capenable & IFCAP_LRO)
1109                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1110                                 else
1111                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1112                         }
1113 #endif
1114                 }
1115 #ifdef TCP_OFFLOAD
1116                 if (mask & IFCAP_TOE) {
1117                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1118
1119                         rc = toe_capability(pi, enable);
1120                         if (rc != 0)
1121                                 goto fail;
1122
1123                         ifp->if_capenable ^= mask;
1124                 }
1125 #endif
1126                 if (mask & IFCAP_VLAN_HWTAGGING) {
1127                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1128                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1129                                 PORT_LOCK(pi);
1130                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1131                                 PORT_UNLOCK(pi);
1132                         }
1133                 }
1134                 if (mask & IFCAP_VLAN_MTU) {
1135                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1136
1137                         /* Need to find out how to disable auto-mtu-inflation */
1138                 }
1139                 if (mask & IFCAP_VLAN_HWTSO)
1140                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1141                 if (mask & IFCAP_VLAN_HWCSUM)
1142                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1143
1144 #ifdef VLAN_CAPABILITIES
1145                 VLAN_CAPABILITIES(ifp);
1146 #endif
1147                 ADAPTER_UNLOCK(sc);
1148                 break;
1149
1150         case SIOCSIFMEDIA:
1151         case SIOCGIFMEDIA:
1152                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1153                 break;
1154
1155         default:
1156                 rc = ether_ioctl(ifp, cmd, data);
1157         }
1158
1159         return (rc);
1160 }
1161
1162 static int
1163 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1164 {
1165         struct port_info *pi = ifp->if_softc;
1166         struct adapter *sc = pi->adapter;
1167         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1168         struct buf_ring *br;
1169         int rc;
1170
1171         M_ASSERTPKTHDR(m);
1172
1173         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1174                 m_freem(m);
1175                 return (ENETDOWN);
1176         }
1177
1178         if (m->m_flags & M_FLOWID)
1179                 txq += (m->m_pkthdr.flowid % pi->ntxq);
1180         br = txq->br;
1181
1182         if (TXQ_TRYLOCK(txq) == 0) {
1183                 struct sge_eq *eq = &txq->eq;
1184
1185                 /*
1186                  * It is possible that t4_eth_tx finishes up and releases the
1187                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1188                  * need to make sure that this mbuf doesn't just sit there in
1189                  * the drbr.
1190                  */
1191
1192                 rc = drbr_enqueue(ifp, br, m);
1193                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1194                     !(eq->flags & EQ_DOOMED))
1195                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1196                 return (rc);
1197         }
1198
1199         /*
1200          * txq->m is the mbuf that is held up due to a temporary shortage of
1201          * resources and it should be put on the wire first.  Then what's in
1202          * drbr and finally the mbuf that was just passed in to us.
1203          *
1204          * Return code should indicate the fate of the mbuf that was passed in
1205          * this time.
1206          */
1207
1208         TXQ_LOCK_ASSERT_OWNED(txq);
1209         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1210
1211                 /* Queued for transmission. */
1212
1213                 rc = drbr_enqueue(ifp, br, m);
1214                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1215                 (void) t4_eth_tx(ifp, txq, m);
1216                 TXQ_UNLOCK(txq);
1217                 return (rc);
1218         }
1219
1220         /* Direct transmission. */
1221         rc = t4_eth_tx(ifp, txq, m);
1222         if (rc != 0 && txq->m)
1223                 rc = 0; /* held, will be transmitted soon (hopefully) */
1224
1225         TXQ_UNLOCK(txq);
1226         return (rc);
1227 }
1228
1229 static void
1230 cxgbe_qflush(struct ifnet *ifp)
1231 {
1232         struct port_info *pi = ifp->if_softc;
1233         struct sge_txq *txq;
1234         int i;
1235         struct mbuf *m;
1236
1237         /* queues do not exist if !PORT_INIT_DONE. */
1238         if (pi->flags & PORT_INIT_DONE) {
1239                 for_each_txq(pi, i, txq) {
1240                         TXQ_LOCK(txq);
1241                         m_freem(txq->m);
1242                         txq->m = NULL;
1243                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1244                                 m_freem(m);
1245                         TXQ_UNLOCK(txq);
1246                 }
1247         }
1248         if_qflush(ifp);
1249 }
1250
1251 static int
1252 cxgbe_media_change(struct ifnet *ifp)
1253 {
1254         struct port_info *pi = ifp->if_softc;
1255
1256         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1257
1258         return (EOPNOTSUPP);
1259 }
1260
1261 static void
1262 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1263 {
1264         struct port_info *pi = ifp->if_softc;
1265         struct ifmedia_entry *cur = pi->media.ifm_cur;
1266         int speed = pi->link_cfg.speed;
1267         int data = (pi->port_type << 8) | pi->mod_type;
1268
1269         if (cur->ifm_data != data) {
1270                 build_medialist(pi);
1271                 cur = pi->media.ifm_cur;
1272         }
1273
1274         ifmr->ifm_status = IFM_AVALID;
1275         if (!pi->link_cfg.link_ok)
1276                 return;
1277
1278         ifmr->ifm_status |= IFM_ACTIVE;
1279
1280         /* active and current will differ iff current media is autoselect. */
1281         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1282                 return;
1283
1284         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1285         if (speed == SPEED_10000)
1286                 ifmr->ifm_active |= IFM_10G_T;
1287         else if (speed == SPEED_1000)
1288                 ifmr->ifm_active |= IFM_1000_T;
1289         else if (speed == SPEED_100)
1290                 ifmr->ifm_active |= IFM_100_TX;
1291         else if (speed == SPEED_10)
1292                 ifmr->ifm_active |= IFM_10_T;
1293         else
1294                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1295                             speed));
1296 }
1297
1298 void
1299 t4_fatal_err(struct adapter *sc)
1300 {
1301         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1302         t4_intr_disable(sc);
1303         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1304             device_get_nameunit(sc->dev));
1305 }
1306
1307 static int
1308 map_bars(struct adapter *sc)
1309 {
1310         sc->regs_rid = PCIR_BAR(0);
1311         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1312             &sc->regs_rid, RF_ACTIVE);
1313         if (sc->regs_res == NULL) {
1314                 device_printf(sc->dev, "cannot map registers.\n");
1315                 return (ENXIO);
1316         }
1317         sc->bt = rman_get_bustag(sc->regs_res);
1318         sc->bh = rman_get_bushandle(sc->regs_res);
1319         sc->mmio_len = rman_get_size(sc->regs_res);
1320
1321         sc->msix_rid = PCIR_BAR(4);
1322         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1323             &sc->msix_rid, RF_ACTIVE);
1324         if (sc->msix_res == NULL) {
1325                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1326                 return (ENXIO);
1327         }
1328
1329         return (0);
1330 }
1331
1332 static void
1333 setup_memwin(struct adapter *sc)
1334 {
1335         uint32_t bar0;
1336
1337         /*
1338          * Read low 32b of bar0 indirectly via the hardware backdoor mechanism.
1339          * Works from within PCI passthrough environments too, where
1340          * rman_get_start() can return a different value.  We need to program
1341          * the memory window decoders with the actual addresses that will be
1342          * coming across the PCIe link.
1343          */
1344         bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1345         bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1346
1347         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1348                      (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1349                      V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1350
1351         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1352                      (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1353                      V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1354
1355         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1356                      (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1357                      V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1358
1359         /* flush */
1360         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1361 }
1362
1363 static int
1364 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1365     struct intrs_and_queues *iaq)
1366 {
1367         int rc, itype, navail, nrxq10g, nrxq1g, n;
1368         int nofldrxq10g = 0, nofldrxq1g = 0;
1369
1370         bzero(iaq, sizeof(*iaq));
1371
1372         iaq->ntxq10g = t4_ntxq10g;
1373         iaq->ntxq1g = t4_ntxq1g;
1374         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1375         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1376 #ifdef TCP_OFFLOAD
1377         if (is_offload(sc)) {
1378                 iaq->nofldtxq10g = t4_nofldtxq10g;
1379                 iaq->nofldtxq1g = t4_nofldtxq1g;
1380                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1381                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1382         }
1383 #endif
1384
1385         for (itype = INTR_MSIX; itype; itype >>= 1) {
1386
1387                 if ((itype & t4_intr_types) == 0)
1388                         continue;       /* not allowed */
1389
1390                 if (itype == INTR_MSIX)
1391                         navail = pci_msix_count(sc->dev);
1392                 else if (itype == INTR_MSI)
1393                         navail = pci_msi_count(sc->dev);
1394                 else
1395                         navail = 1;
1396 restart:
1397                 if (navail == 0)
1398                         continue;
1399
1400                 iaq->intr_type = itype;
1401                 iaq->intr_flags = 0;
1402
1403                 /*
1404                  * Best option: an interrupt vector for errors, one for the
1405                  * firmware event queue, and one each for each rxq (NIC as well
1406                  * as offload).
1407                  */
1408                 iaq->nirq = T4_EXTRA_INTR;
1409                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1410                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1411                 if (iaq->nirq <= navail &&
1412                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1413                         iaq->intr_flags |= INTR_DIRECT;
1414                         goto allocate;
1415                 }
1416
1417                 /*
1418                  * Second best option: an interrupt vector for errors, one for
1419                  * the firmware event queue, and one each for either NIC or
1420                  * offload rxq's.
1421                  */
1422                 iaq->nirq = T4_EXTRA_INTR;
1423                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1424                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1425                 if (iaq->nirq <= navail &&
1426                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1427                         goto allocate;
1428
1429                 /*
1430                  * Next best option: an interrupt vector for errors, one for the
1431                  * firmware event queue, and at least one per port.  At this
1432                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1433                  * what's available to us.
1434                  */
1435                 iaq->nirq = T4_EXTRA_INTR;
1436                 iaq->nirq += n10g + n1g;
1437                 if (iaq->nirq <= navail) {
1438                         int leftover = navail - iaq->nirq;
1439
1440                         if (n10g > 0) {
1441                                 int target = max(nrxq10g, nofldrxq10g);
1442
1443                                 n = 1;
1444                                 while (n < target && leftover >= n10g) {
1445                                         leftover -= n10g;
1446                                         iaq->nirq += n10g;
1447                                         n++;
1448                                 }
1449                                 iaq->nrxq10g = min(n, nrxq10g);
1450 #ifdef TCP_OFFLOAD
1451                                 if (is_offload(sc))
1452                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1453 #endif
1454                         }
1455
1456                         if (n1g > 0) {
1457                                 int target = max(nrxq1g, nofldrxq1g);
1458
1459                                 n = 1;
1460                                 while (n < target && leftover >= n1g) {
1461                                         leftover -= n1g;
1462                                         iaq->nirq += n1g;
1463                                         n++;
1464                                 }
1465                                 iaq->nrxq1g = min(n, nrxq1g);
1466 #ifdef TCP_OFFLOAD
1467                                 if (is_offload(sc))
1468                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1469 #endif
1470                         }
1471
1472                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1473                                 goto allocate;
1474                 }
1475
1476                 /*
1477                  * Least desirable option: one interrupt vector for everything.
1478                  */
1479                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1480 #ifdef TCP_OFFLOAD
1481                 if (is_offload(sc))
1482                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1483 #endif
1484
1485 allocate:
1486                 navail = iaq->nirq;
1487                 rc = 0;
1488                 if (itype == INTR_MSIX)
1489                         rc = pci_alloc_msix(sc->dev, &navail);
1490                 else if (itype == INTR_MSI)
1491                         rc = pci_alloc_msi(sc->dev, &navail);
1492
1493                 if (rc == 0) {
1494                         if (navail == iaq->nirq)
1495                                 return (0);
1496
1497                         /*
1498                          * Didn't get the number requested.  Use whatever number
1499                          * the kernel is willing to allocate (it's in navail).
1500                          */
1501                         device_printf(sc->dev, "fewer vectors than requested, "
1502                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1503                             itype, iaq->nirq, navail);
1504                         pci_release_msi(sc->dev);
1505                         goto restart;
1506                 }
1507
1508                 device_printf(sc->dev,
1509                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1510                     itype, rc, iaq->nirq, navail);
1511         }
1512
1513         device_printf(sc->dev,
1514             "failed to find a usable interrupt type.  "
1515             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1516             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1517
1518         return (ENXIO);
1519 }
1520
1521 /*
1522  * Install a compatible firmware (if required), establish contact with it (by
1523  * saying hello), and reset the device.  If we end up as the master driver,
1524  * partition adapter resources by providing a configuration file to the
1525  * firmware.
1526  */
1527 static int
1528 prep_firmware(struct adapter *sc)
1529 {
1530         const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1531         int rc;
1532         enum dev_state state;
1533
1534         default_cfg = firmware_get(T4_CFGNAME);
1535
1536         /* Check firmware version and install a different one if necessary */
1537         rc = t4_check_fw_version(sc);
1538         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1539             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1540             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1541             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1542             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1543         if (rc != 0) {
1544                 uint32_t v = 0;
1545
1546                 fw = firmware_get(T4_FWNAME);
1547                 if (fw != NULL) {
1548                         const struct fw_hdr *hdr = (const void *)fw->data;
1549
1550                         v = ntohl(hdr->fw_ver);
1551
1552                         /*
1553                          * The firmware module will not be used if it isn't the
1554                          * same major version as what the driver was compiled
1555                          * with.
1556                          */
1557                         if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
1558                                 device_printf(sc->dev,
1559                                     "Found firmware image but version %d "
1560                                     "can not be used with this driver (%d)\n",
1561                                     G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR);
1562
1563                                 firmware_put(fw, FIRMWARE_UNLOAD);
1564                                 fw = NULL;
1565                         }
1566                 }
1567
1568                 if (fw == NULL && rc < 0) {
1569                         device_printf(sc->dev, "No usable firmware. "
1570                             "card has %d.%d.%d, driver compiled with %d.%d.%d",
1571                             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1572                             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1573                             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1574                             FW_VERSION_MAJOR, FW_VERSION_MINOR,
1575                             FW_VERSION_MICRO);
1576                         rc = EAGAIN;
1577                         goto done;
1578                 }
1579
1580                 /*
1581                  * Always upgrade, even for minor/micro/build mismatches.
1582                  * Downgrade only for a major version mismatch or if
1583                  * force_firmware_install was specified.
1584                  */
1585                 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
1586                         device_printf(sc->dev,
1587                             "installing firmware %d.%d.%d.%d on card.\n",
1588                             G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1589                             G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1590
1591                         rc = -t4_load_fw(sc, fw->data, fw->datasize);
1592                         if (rc != 0) {
1593                                 device_printf(sc->dev,
1594                                     "failed to install firmware: %d\n", rc);
1595                                 goto done;
1596                         } else {
1597                                 /* refresh */
1598                                 (void) t4_check_fw_version(sc);
1599                                 snprintf(sc->fw_version,
1600                                     sizeof(sc->fw_version), "%u.%u.%u.%u",
1601                                     G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1602                                     G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1603                                     G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1604                                     G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1605                         }
1606                 }
1607         }
1608
1609         /* Contact firmware.  */
1610         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1611         if (rc < 0) {
1612                 rc = -rc;
1613                 device_printf(sc->dev,
1614                     "failed to connect to the firmware: %d.\n", rc);
1615                 goto done;
1616         }
1617         if (rc == sc->mbox)
1618                 sc->flags |= MASTER_PF;
1619
1620         /* Reset device */
1621         rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1622         if (rc != 0) {
1623                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1624                 if (rc != ETIMEDOUT && rc != EIO)
1625                         t4_fw_bye(sc, sc->mbox);
1626                 goto done;
1627         }
1628
1629         /* Partition adapter resources as specified in the config file. */
1630         if (sc->flags & MASTER_PF) {
1631                 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
1632                         char s[32];
1633
1634                         snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
1635                         cfg = firmware_get(s);
1636                         if (cfg == NULL) {
1637                                 device_printf(sc->dev,
1638                                     "unable to locate %s module, "
1639                                     "will use default config file.\n", s);
1640                         }
1641                 }
1642
1643                 rc = partition_resources(sc, cfg ? cfg : default_cfg);
1644                 if (rc != 0)
1645                         goto done;      /* error message displayed already */
1646         }
1647
1648         sc->flags |= FW_OK;
1649
1650 done:
1651         if (fw != NULL)
1652                 firmware_put(fw, FIRMWARE_UNLOAD);
1653         if (cfg != NULL)
1654                 firmware_put(cfg, FIRMWARE_UNLOAD);
1655         if (default_cfg != NULL)
1656                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
1657
1658         return (rc);
1659 }
1660
1661 #define FW_PARAM_DEV(param) \
1662         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1663          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1664 #define FW_PARAM_PFVF(param) \
1665         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1666          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1667
1668 /*
1669  * Upload configuration file to card's memory.
1670  */
1671 static int
1672 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1673     uint32_t *ma)
1674 {
1675         int rc, i;
1676         uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1677         const uint32_t *b;
1678
1679         /* Figure out where the firmware wants us to upload it. */
1680         param = FW_PARAM_DEV(CF);
1681         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1682         if (rc != 0) {
1683                 /* Firmwares without config file support will fail this way */
1684                 device_printf(sc->dev,
1685                     "failed to query config file location: %d.\n", rc);
1686                 return (rc);
1687         }
1688         *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1689         *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1690
1691         if (maddr & 3) {
1692                 device_printf(sc->dev,
1693                     "cannot upload config file (type %u, addr %x).\n",
1694                     mtype, maddr);
1695                 return (EFAULT);
1696         }
1697
1698         /* Translate mtype/maddr to an address suitable for the PCIe window */
1699         val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1700         val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1701         switch (mtype) {
1702         case FW_MEMTYPE_CF_EDC0:
1703                 if (!(val & F_EDRAM0_ENABLE))
1704                         goto err;
1705                 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1706                 maddr += G_EDRAM0_BASE(bar) << 20;
1707                 break;
1708
1709         case FW_MEMTYPE_CF_EDC1:
1710                 if (!(val & F_EDRAM1_ENABLE))
1711                         goto err;
1712                 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1713                 maddr += G_EDRAM1_BASE(bar) << 20;
1714                 break;
1715
1716         case FW_MEMTYPE_CF_EXTMEM:
1717                 if (!(val & F_EXT_MEM_ENABLE))
1718                         goto err;
1719                 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1720                 maddr += G_EXT_MEM_BASE(bar) << 20;
1721                 break;
1722
1723         default:
1724 err:
1725                 device_printf(sc->dev,
1726                     "cannot upload config file (type %u, enabled %u).\n",
1727                     mtype, val);
1728                 return (EFAULT);
1729         }
1730
1731         /*
1732          * Position the PCIe window (we use memwin2) to the 16B aligned area
1733          * just at/before the upload location.
1734          */
1735         win = maddr & ~0xf;
1736         off = maddr - win;  /* offset from the start of the window. */
1737         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1738         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1739
1740         remaining = fw->datasize;
1741         if (remaining > FLASH_CFG_MAX_SIZE ||
1742             remaining > MEMWIN2_APERTURE - off) {
1743                 device_printf(sc->dev, "cannot upload config file all at once "
1744                     "(size %u, max %u, room %u).\n",
1745                     remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1746                 return (EFBIG);
1747         }
1748
1749         /*
1750          * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1751          * stuffing/comments at the end of the config file so it's ok to simply
1752          * throw away the last remaining bytes when the config file is not an
1753          * exact multiple of 4.
1754          */
1755         b = fw->data;
1756         for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1757                 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1758
1759         return (rc);
1760 }
1761
1762 /*
1763  * Partition chip resources for use between various PFs, VFs, etc.  This is done
1764  * by uploading the firmware configuration file to the adapter and instructing
1765  * the firmware to process it.
1766  */
1767 static int
1768 partition_resources(struct adapter *sc, const struct firmware *cfg)
1769 {
1770         int rc;
1771         struct fw_caps_config_cmd caps;
1772         uint32_t mtype, maddr, finicsum, cfcsum;
1773
1774         rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1775         if (rc != 0) {
1776                 mtype = FW_MEMTYPE_CF_FLASH;
1777                 maddr = t4_flash_cfg_addr(sc);
1778         }
1779
1780         bzero(&caps, sizeof(caps));
1781         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1782             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1783         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1784             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1785             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1786         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1787         if (rc != 0) {
1788                 device_printf(sc->dev,
1789                     "failed to pre-process config file: %d.\n", rc);
1790                 return (rc);
1791         }
1792
1793         finicsum = be32toh(caps.finicsum);
1794         cfcsum = be32toh(caps.cfcsum);
1795         if (finicsum != cfcsum) {
1796                 device_printf(sc->dev,
1797                     "WARNING: config file checksum mismatch: %08x %08x\n",
1798                     finicsum, cfcsum);
1799         }
1800         sc->cfcsum = cfcsum;
1801
1802 #define LIMIT_CAPS(x) do { \
1803         caps.x &= htobe16(t4_##x##_allowed); \
1804         sc->x = htobe16(caps.x); \
1805 } while (0)
1806
1807         /*
1808          * Let the firmware know what features will (not) be used so it can tune
1809          * things accordingly.
1810          */
1811         LIMIT_CAPS(linkcaps);
1812         LIMIT_CAPS(niccaps);
1813         LIMIT_CAPS(toecaps);
1814         LIMIT_CAPS(rdmacaps);
1815         LIMIT_CAPS(iscsicaps);
1816         LIMIT_CAPS(fcoecaps);
1817 #undef LIMIT_CAPS
1818
1819         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1820             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1821         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1822         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1823         if (rc != 0) {
1824                 device_printf(sc->dev,
1825                     "failed to process config file: %d.\n", rc);
1826                 return (rc);
1827         }
1828
1829         return (0);
1830 }
1831
1832 /*
1833  * Retrieve parameters that are needed (or nice to have) prior to calling
1834  * t4_sge_init and t4_fw_initialize.
1835  */
1836 static int
1837 get_params__pre_init(struct adapter *sc)
1838 {
1839         int rc;
1840         uint32_t param[2], val[2];
1841         struct fw_devlog_cmd cmd;
1842         struct devlog_params *dlog = &sc->params.devlog;
1843
1844         param[0] = FW_PARAM_DEV(PORTVEC);
1845         param[1] = FW_PARAM_DEV(CCLK);
1846         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1847         if (rc != 0) {
1848                 device_printf(sc->dev,
1849                     "failed to query parameters (pre_init): %d.\n", rc);
1850                 return (rc);
1851         }
1852
1853         sc->params.portvec = val[0];
1854         sc->params.nports = bitcount32(val[0]);
1855         sc->params.vpd.cclk = val[1];
1856
1857         /* Read device log parameters. */
1858         bzero(&cmd, sizeof(cmd));
1859         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1860             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1861         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1862         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1863         if (rc != 0) {
1864                 device_printf(sc->dev,
1865                     "failed to get devlog parameters: %d.\n", rc);
1866                 bzero(dlog, sizeof (*dlog));
1867                 rc = 0; /* devlog isn't critical for device operation */
1868         } else {
1869                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1870                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1871                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1872                 dlog->size = be32toh(cmd.memsize_devlog);
1873         }
1874
1875         return (rc);
1876 }
1877
1878 /*
1879  * Retrieve various parameters that are of interest to the driver.  The device
1880  * has been initialized by the firmware at this point.
1881  */
1882 static int
1883 get_params__post_init(struct adapter *sc)
1884 {
1885         int rc;
1886         uint32_t param[7], val[7];
1887         struct fw_caps_config_cmd caps;
1888
1889         param[0] = FW_PARAM_PFVF(IQFLINT_START);
1890         param[1] = FW_PARAM_PFVF(EQ_START);
1891         param[2] = FW_PARAM_PFVF(FILTER_START);
1892         param[3] = FW_PARAM_PFVF(FILTER_END);
1893         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
1894         if (rc != 0) {
1895                 device_printf(sc->dev,
1896                     "failed to query parameters (post_init): %d.\n", rc);
1897                 return (rc);
1898         }
1899
1900         sc->sge.iq_start = val[0];
1901         sc->sge.eq_start = val[1];
1902         sc->tids.ftid_base = val[2];
1903         sc->tids.nftids = val[3] - val[2] + 1;
1904
1905         /* get capabilites */
1906         bzero(&caps, sizeof(caps));
1907         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1908             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1909         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1910         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1911         if (rc != 0) {
1912                 device_printf(sc->dev,
1913                     "failed to get card capabilities: %d.\n", rc);
1914                 return (rc);
1915         }
1916
1917         if (caps.toecaps) {
1918                 /* query offload-related parameters */
1919                 param[0] = FW_PARAM_DEV(NTID);
1920                 param[1] = FW_PARAM_PFVF(SERVER_START);
1921                 param[2] = FW_PARAM_PFVF(SERVER_END);
1922                 param[3] = FW_PARAM_PFVF(TDDP_START);
1923                 param[4] = FW_PARAM_PFVF(TDDP_END);
1924                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1925                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1926                 if (rc != 0) {
1927                         device_printf(sc->dev,
1928                             "failed to query TOE parameters: %d.\n", rc);
1929                         return (rc);
1930                 }
1931                 sc->tids.ntids = val[0];
1932                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1933                 sc->tids.stid_base = val[1];
1934                 sc->tids.nstids = val[2] - val[1] + 1;
1935                 sc->vres.ddp.start = val[3];
1936                 sc->vres.ddp.size = val[4] - val[3] + 1;
1937                 sc->params.ofldq_wr_cred = val[5];
1938                 sc->params.offload = 1;
1939         }
1940         if (caps.rdmacaps) {
1941                 param[0] = FW_PARAM_PFVF(STAG_START);
1942                 param[1] = FW_PARAM_PFVF(STAG_END);
1943                 param[2] = FW_PARAM_PFVF(RQ_START);
1944                 param[3] = FW_PARAM_PFVF(RQ_END);
1945                 param[4] = FW_PARAM_PFVF(PBL_START);
1946                 param[5] = FW_PARAM_PFVF(PBL_END);
1947                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1948                 if (rc != 0) {
1949                         device_printf(sc->dev,
1950                             "failed to query RDMA parameters(1): %d.\n", rc);
1951                         return (rc);
1952                 }
1953                 sc->vres.stag.start = val[0];
1954                 sc->vres.stag.size = val[1] - val[0] + 1;
1955                 sc->vres.rq.start = val[2];
1956                 sc->vres.rq.size = val[3] - val[2] + 1;
1957                 sc->vres.pbl.start = val[4];
1958                 sc->vres.pbl.size = val[5] - val[4] + 1;
1959
1960                 param[0] = FW_PARAM_PFVF(SQRQ_START);
1961                 param[1] = FW_PARAM_PFVF(SQRQ_END);
1962                 param[2] = FW_PARAM_PFVF(CQ_START);
1963                 param[3] = FW_PARAM_PFVF(CQ_END);
1964                 param[4] = FW_PARAM_PFVF(OCQ_START);
1965                 param[5] = FW_PARAM_PFVF(OCQ_END);
1966                 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
1967                 if (rc != 0) {
1968                         device_printf(sc->dev,
1969                             "failed to query RDMA parameters(2): %d.\n", rc);
1970                         return (rc);
1971                 }
1972                 sc->vres.qp.start = val[0];
1973                 sc->vres.qp.size = val[1] - val[0] + 1;
1974                 sc->vres.cq.start = val[2];
1975                 sc->vres.cq.size = val[3] - val[2] + 1;
1976                 sc->vres.ocq.start = val[4];
1977                 sc->vres.ocq.size = val[5] - val[4] + 1;
1978         }
1979         if (caps.iscsicaps) {
1980                 param[0] = FW_PARAM_PFVF(ISCSI_START);
1981                 param[1] = FW_PARAM_PFVF(ISCSI_END);
1982                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1983                 if (rc != 0) {
1984                         device_printf(sc->dev,
1985                             "failed to query iSCSI parameters: %d.\n", rc);
1986                         return (rc);
1987                 }
1988                 sc->vres.iscsi.start = val[0];
1989                 sc->vres.iscsi.size = val[1] - val[0] + 1;
1990         }
1991
1992         /* These are finalized by FW initialization, load their values now */
1993         val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1994         sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1995         sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1996         t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1997
1998         return (rc);
1999 }
2000
2001 #undef FW_PARAM_PFVF
2002 #undef FW_PARAM_DEV
2003
2004 static void
2005 t4_set_desc(struct adapter *sc)
2006 {
2007         char buf[128];
2008         struct adapter_params *p = &sc->params;
2009
2010         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
2011             p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
2012
2013         device_set_desc_copy(sc->dev, buf);
2014 }
2015
2016 static void
2017 build_medialist(struct port_info *pi)
2018 {
2019         struct ifmedia *media = &pi->media;
2020         int data, m;
2021
2022         PORT_LOCK(pi);
2023
2024         ifmedia_removeall(media);
2025
2026         m = IFM_ETHER | IFM_FDX;
2027         data = (pi->port_type << 8) | pi->mod_type;
2028
2029         switch(pi->port_type) {
2030         case FW_PORT_TYPE_BT_XFI:
2031                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2032                 break;
2033
2034         case FW_PORT_TYPE_BT_XAUI:
2035                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2036                 /* fall through */
2037
2038         case FW_PORT_TYPE_BT_SGMII:
2039                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2040                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2041                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2042                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2043                 break;
2044
2045         case FW_PORT_TYPE_CX4:
2046                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2047                 ifmedia_set(media, m | IFM_10G_CX4);
2048                 break;
2049
2050         case FW_PORT_TYPE_SFP:
2051         case FW_PORT_TYPE_FIBER_XFI:
2052         case FW_PORT_TYPE_FIBER_XAUI:
2053                 switch (pi->mod_type) {
2054
2055                 case FW_PORT_MOD_TYPE_LR:
2056                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2057                         ifmedia_set(media, m | IFM_10G_LR);
2058                         break;
2059
2060                 case FW_PORT_MOD_TYPE_SR:
2061                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2062                         ifmedia_set(media, m | IFM_10G_SR);
2063                         break;
2064
2065                 case FW_PORT_MOD_TYPE_LRM:
2066                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2067                         ifmedia_set(media, m | IFM_10G_LRM);
2068                         break;
2069
2070                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2071                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2072                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2073                         ifmedia_set(media, m | IFM_10G_TWINAX);
2074                         break;
2075
2076                 case FW_PORT_MOD_TYPE_NONE:
2077                         m &= ~IFM_FDX;
2078                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2079                         ifmedia_set(media, m | IFM_NONE);
2080                         break;
2081
2082                 case FW_PORT_MOD_TYPE_NA:
2083                 case FW_PORT_MOD_TYPE_ER:
2084                 default:
2085                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2086                         ifmedia_set(media, m | IFM_UNKNOWN);
2087                         break;
2088                 }
2089                 break;
2090
2091         case FW_PORT_TYPE_KX4:
2092         case FW_PORT_TYPE_KX:
2093         case FW_PORT_TYPE_KR:
2094         default:
2095                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2096                 ifmedia_set(media, m | IFM_UNKNOWN);
2097                 break;
2098         }
2099
2100         PORT_UNLOCK(pi);
2101 }
2102
2103 #define FW_MAC_EXACT_CHUNK      7
2104
2105 /*
2106  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2107  * indicates which parameters should be programmed (the rest are left alone).
2108  */
2109 static int
2110 update_mac_settings(struct port_info *pi, int flags)
2111 {
2112         int rc;
2113         struct ifnet *ifp = pi->ifp;
2114         struct adapter *sc = pi->adapter;
2115         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2116
2117         PORT_LOCK_ASSERT_OWNED(pi);
2118         KASSERT(flags, ("%s: not told what to update.", __func__));
2119
2120         if (flags & XGMAC_MTU)
2121                 mtu = ifp->if_mtu;
2122
2123         if (flags & XGMAC_PROMISC)
2124                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2125
2126         if (flags & XGMAC_ALLMULTI)
2127                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2128
2129         if (flags & XGMAC_VLANEX)
2130                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2131
2132         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2133             vlanex, false);
2134         if (rc) {
2135                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2136                 return (rc);
2137         }
2138
2139         if (flags & XGMAC_UCADDR) {
2140                 uint8_t ucaddr[ETHER_ADDR_LEN];
2141
2142                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2143                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2144                     ucaddr, true, true);
2145                 if (rc < 0) {
2146                         rc = -rc;
2147                         if_printf(ifp, "change_mac failed: %d\n", rc);
2148                         return (rc);
2149                 } else {
2150                         pi->xact_addr_filt = rc;
2151                         rc = 0;
2152                 }
2153         }
2154
2155         if (flags & XGMAC_MCADDRS) {
2156                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2157                 int del = 1;
2158                 uint64_t hash = 0;
2159                 struct ifmultiaddr *ifma;
2160                 int i = 0, j;
2161
2162                 if_maddr_rlock(ifp);
2163                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2164                         if (ifma->ifma_addr->sa_family != AF_LINK)
2165                                 continue;
2166                         mcaddr[i++] =
2167                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2168
2169                         if (i == FW_MAC_EXACT_CHUNK) {
2170                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2171                                     del, i, mcaddr, NULL, &hash, 0);
2172                                 if (rc < 0) {
2173                                         rc = -rc;
2174                                         for (j = 0; j < i; j++) {
2175                                                 if_printf(ifp,
2176                                                     "failed to add mc address"
2177                                                     " %02x:%02x:%02x:"
2178                                                     "%02x:%02x:%02x rc=%d\n",
2179                                                     mcaddr[j][0], mcaddr[j][1],
2180                                                     mcaddr[j][2], mcaddr[j][3],
2181                                                     mcaddr[j][4], mcaddr[j][5],
2182                                                     rc);
2183                                         }
2184                                         goto mcfail;
2185                                 }
2186                                 del = 0;
2187                                 i = 0;
2188                         }
2189                 }
2190                 if (i > 0) {
2191                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2192                             del, i, mcaddr, NULL, &hash, 0);
2193                         if (rc < 0) {
2194                                 rc = -rc;
2195                                 for (j = 0; j < i; j++) {
2196                                         if_printf(ifp,
2197                                             "failed to add mc address"
2198                                             " %02x:%02x:%02x:"
2199                                             "%02x:%02x:%02x rc=%d\n",
2200                                             mcaddr[j][0], mcaddr[j][1],
2201                                             mcaddr[j][2], mcaddr[j][3],
2202                                             mcaddr[j][4], mcaddr[j][5],
2203                                             rc);
2204                                 }
2205                                 goto mcfail;
2206                         }
2207                 }
2208
2209                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2210                 if (rc != 0)
2211                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2212 mcfail:
2213                 if_maddr_runlock(ifp);
2214         }
2215
2216         return (rc);
2217 }
2218
2219 static int
2220 cxgbe_init_locked(struct port_info *pi)
2221 {
2222         struct adapter *sc = pi->adapter;
2223         int rc = 0;
2224
2225         ADAPTER_LOCK_ASSERT_OWNED(sc);
2226
2227         while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2228                 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) {
2229                         rc = EINTR;
2230                         goto done;
2231                 }
2232         }
2233         if (IS_DOOMED(pi)) {
2234                 rc = ENXIO;
2235                 goto done;
2236         }
2237         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2238
2239         /* Give up the adapter lock, port init code can sleep. */
2240         SET_BUSY(sc);
2241         ADAPTER_UNLOCK(sc);
2242
2243         rc = cxgbe_init_synchronized(pi);
2244
2245 done:
2246         ADAPTER_LOCK(sc);
2247         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2248         CLR_BUSY(sc);
2249         wakeup_one(&sc->flags);
2250         ADAPTER_UNLOCK(sc);
2251         return (rc);
2252 }
2253
2254 static int
2255 cxgbe_init_synchronized(struct port_info *pi)
2256 {
2257         struct adapter *sc = pi->adapter;
2258         struct ifnet *ifp = pi->ifp;
2259         int rc = 0;
2260
2261         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2262
2263         if (isset(&sc->open_device_map, pi->port_id)) {
2264                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2265                     ("mismatch between open_device_map and if_drv_flags"));
2266                 return (0);     /* already running */
2267         }
2268
2269         if (!(sc->flags & FULL_INIT_DONE) &&
2270             ((rc = adapter_full_init(sc)) != 0))
2271                 return (rc);    /* error message displayed already */
2272
2273         if (!(pi->flags & PORT_INIT_DONE) &&
2274             ((rc = port_full_init(pi)) != 0))
2275                 return (rc); /* error message displayed already */
2276
2277         PORT_LOCK(pi);
2278         rc = update_mac_settings(pi, XGMAC_ALL);
2279         PORT_UNLOCK(pi);
2280         if (rc)
2281                 goto done;      /* error message displayed already */
2282
2283         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2284         if (rc != 0) {
2285                 if_printf(ifp, "start_link failed: %d\n", rc);
2286                 goto done;
2287         }
2288
2289         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2290         if (rc != 0) {
2291                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2292                 goto done;
2293         }
2294
2295         /* all ok */
2296         setbit(&sc->open_device_map, pi->port_id);
2297         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2298
2299         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2300 done:
2301         if (rc != 0)
2302                 cxgbe_uninit_synchronized(pi);
2303
2304         return (rc);
2305 }
2306
2307 static int
2308 cxgbe_uninit_locked(struct port_info *pi)
2309 {
2310         struct adapter *sc = pi->adapter;
2311         int rc;
2312
2313         ADAPTER_LOCK_ASSERT_OWNED(sc);
2314
2315         while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2316                 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) {
2317                         rc = EINTR;
2318                         goto done;
2319                 }
2320         }
2321         if (IS_DOOMED(pi)) {
2322                 rc = ENXIO;
2323                 goto done;
2324         }
2325         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2326         SET_BUSY(sc);
2327         ADAPTER_UNLOCK(sc);
2328
2329         rc = cxgbe_uninit_synchronized(pi);
2330
2331         ADAPTER_LOCK(sc);
2332         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2333         CLR_BUSY(sc);
2334         wakeup_one(&sc->flags);
2335 done:
2336         ADAPTER_UNLOCK(sc);
2337         return (rc);
2338 }
2339
2340 /*
2341  * Idempotent.
2342  */
2343 static int
2344 cxgbe_uninit_synchronized(struct port_info *pi)
2345 {
2346         struct adapter *sc = pi->adapter;
2347         struct ifnet *ifp = pi->ifp;
2348         int rc;
2349
2350         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2351
2352         /*
2353          * Disable the VI so that all its data in either direction is discarded
2354          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2355          * tick) intact as the TP can deliver negative advice or data that it's
2356          * holding in its RAM (for an offloaded connection) even after the VI is
2357          * disabled.
2358          */
2359         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2360         if (rc) {
2361                 if_printf(ifp, "disable_vi failed: %d\n", rc);
2362                 return (rc);
2363         }
2364
2365         clrbit(&sc->open_device_map, pi->port_id);
2366         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2367
2368         pi->link_cfg.link_ok = 0;
2369         pi->link_cfg.speed = 0;
2370         t4_os_link_changed(sc, pi->port_id, 0);
2371
2372         return (0);
2373 }
2374
2375 /*
2376  * It is ok for this function to fail midway and return right away.  t4_detach
2377  * will walk the entire sc->irq list and clean up whatever is valid.
2378  */
2379 static int
2380 setup_intr_handlers(struct adapter *sc)
2381 {
2382         int rc, rid, p, q;
2383         char s[8];
2384         struct irq *irq;
2385         struct port_info *pi;
2386         struct sge_rxq *rxq;
2387 #ifdef TCP_OFFLOAD
2388         struct sge_ofld_rxq *ofld_rxq;
2389 #endif
2390
2391         /*
2392          * Setup interrupts.
2393          */
2394         irq = &sc->irq[0];
2395         rid = sc->intr_type == INTR_INTX ? 0 : 1;
2396         if (sc->intr_count == 1) {
2397                 KASSERT(!(sc->flags & INTR_DIRECT),
2398                     ("%s: single interrupt && INTR_DIRECT?", __func__));
2399
2400                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
2401                 if (rc != 0)
2402                         return (rc);
2403         } else {
2404                 /* Multiple interrupts. */
2405                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2406                     ("%s: too few intr.", __func__));
2407
2408                 /* The first one is always error intr */
2409                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
2410                 if (rc != 0)
2411                         return (rc);
2412                 irq++;
2413                 rid++;
2414
2415                 /* The second one is always the firmware event queue */
2416                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
2417                     "evt");
2418                 if (rc != 0)
2419                         return (rc);
2420                 irq++;
2421                 rid++;
2422
2423                 /*
2424                  * Note that if INTR_DIRECT is not set then either the NIC rx
2425                  * queues or (exclusive or) the TOE rx queueus will be taking
2426                  * direct interrupts.
2427                  *
2428                  * There is no need to check for is_offload(sc) as nofldrxq
2429                  * will be 0 if offload is disabled.
2430                  */
2431                 for_each_port(sc, p) {
2432                         pi = sc->port[p];
2433
2434 #ifdef TCP_OFFLOAD
2435                         /*
2436                          * Skip over the NIC queues if they aren't taking direct
2437                          * interrupts.
2438                          */
2439                         if (!(sc->flags & INTR_DIRECT) &&
2440                             pi->nofldrxq > pi->nrxq)
2441                                 goto ofld_queues;
2442 #endif
2443                         rxq = &sc->sge.rxq[pi->first_rxq];
2444                         for (q = 0; q < pi->nrxq; q++, rxq++) {
2445                                 snprintf(s, sizeof(s), "%d.%d", p, q);
2446                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
2447                                     s);
2448                                 if (rc != 0)
2449                                         return (rc);
2450                                 irq++;
2451                                 rid++;
2452                         }
2453
2454 #ifdef TCP_OFFLOAD
2455                         /*
2456                          * Skip over the offload queues if they aren't taking
2457                          * direct interrupts.
2458                          */
2459                         if (!(sc->flags & INTR_DIRECT))
2460                                 continue;
2461 ofld_queues:
2462                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2463                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2464                                 snprintf(s, sizeof(s), "%d,%d", p, q);
2465                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
2466                                     ofld_rxq, s);
2467                                 if (rc != 0)
2468                                         return (rc);
2469                                 irq++;
2470                                 rid++;
2471                         }
2472 #endif
2473                 }
2474         }
2475
2476         return (0);
2477 }
2478
2479 static int
2480 adapter_full_init(struct adapter *sc)
2481 {
2482         int rc, i;
2483
2484         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2485         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2486             ("%s: FULL_INIT_DONE already", __func__));
2487
2488         /*
2489          * queues that belong to the adapter (not any particular port).
2490          */
2491         rc = t4_setup_adapter_queues(sc);
2492         if (rc != 0)
2493                 goto done;
2494
2495         for (i = 0; i < nitems(sc->tq); i++) {
2496                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2497                     taskqueue_thread_enqueue, &sc->tq[i]);
2498                 if (sc->tq[i] == NULL) {
2499                         device_printf(sc->dev,
2500                             "failed to allocate task queue %d\n", i);
2501                         rc = ENOMEM;
2502                         goto done;
2503                 }
2504                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2505                     device_get_nameunit(sc->dev), i);
2506         }
2507
2508         t4_intr_enable(sc);
2509         sc->flags |= FULL_INIT_DONE;
2510 done:
2511         if (rc != 0)
2512                 adapter_full_uninit(sc);
2513
2514         return (rc);
2515 }
2516
2517 static int
2518 adapter_full_uninit(struct adapter *sc)
2519 {
2520         int i;
2521
2522         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2523
2524         t4_teardown_adapter_queues(sc);
2525
2526         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
2527                 taskqueue_free(sc->tq[i]);
2528                 sc->tq[i] = NULL;
2529         }
2530
2531         sc->flags &= ~FULL_INIT_DONE;
2532
2533         return (0);
2534 }
2535
2536 static int
2537 port_full_init(struct port_info *pi)
2538 {
2539         struct adapter *sc = pi->adapter;
2540         struct ifnet *ifp = pi->ifp;
2541         uint16_t *rss;
2542         struct sge_rxq *rxq;
2543         int rc, i;
2544
2545         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2546         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2547             ("%s: PORT_INIT_DONE already", __func__));
2548
2549         sysctl_ctx_init(&pi->ctx);
2550         pi->flags |= PORT_SYSCTL_CTX;
2551
2552         /*
2553          * Allocate tx/rx/fl queues for this port.
2554          */
2555         rc = t4_setup_port_queues(pi);
2556         if (rc != 0)
2557                 goto done;      /* error message displayed already */
2558
2559         /*
2560          * Setup RSS for this port.
2561          */
2562         rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2563             M_ZERO | M_WAITOK);
2564         for_each_rxq(pi, i, rxq) {
2565                 rss[i] = rxq->iq.abs_id;
2566         }
2567         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2568             pi->rss_size, rss, pi->nrxq);
2569         free(rss, M_CXGBE);
2570         if (rc != 0) {
2571                 if_printf(ifp, "rss_config failed: %d\n", rc);
2572                 goto done;
2573         }
2574
2575         pi->flags |= PORT_INIT_DONE;
2576 done:
2577         if (rc != 0)
2578                 port_full_uninit(pi);
2579
2580         return (rc);
2581 }
2582
2583 /*
2584  * Idempotent.
2585  */
2586 static int
2587 port_full_uninit(struct port_info *pi)
2588 {
2589         struct adapter *sc = pi->adapter;
2590         int i;
2591         struct sge_rxq *rxq;
2592         struct sge_txq *txq;
2593 #ifdef TCP_OFFLOAD
2594         struct sge_ofld_rxq *ofld_rxq;
2595         struct sge_wrq *ofld_txq;
2596 #endif
2597
2598         if (pi->flags & PORT_INIT_DONE) {
2599
2600                 /* Need to quiesce queues.  XXX: ctrl queues? */
2601
2602                 for_each_txq(pi, i, txq) {
2603                         quiesce_eq(sc, &txq->eq);
2604                 }
2605
2606 #ifdef TCP_OFFLOAD
2607                 for_each_ofld_txq(pi, i, ofld_txq) {
2608                         quiesce_eq(sc, &ofld_txq->eq);
2609                 }
2610 #endif
2611
2612                 for_each_rxq(pi, i, rxq) {
2613                         quiesce_iq(sc, &rxq->iq);
2614                         quiesce_fl(sc, &rxq->fl);
2615                 }
2616
2617 #ifdef TCP_OFFLOAD
2618                 for_each_ofld_rxq(pi, i, ofld_rxq) {
2619                         quiesce_iq(sc, &ofld_rxq->iq);
2620                         quiesce_fl(sc, &ofld_rxq->fl);
2621                 }
2622 #endif
2623         }
2624
2625         t4_teardown_port_queues(pi);
2626         pi->flags &= ~PORT_INIT_DONE;
2627
2628         return (0);
2629 }
2630
2631 static void
2632 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
2633 {
2634         EQ_LOCK(eq);
2635         eq->flags |= EQ_DOOMED;
2636
2637         /*
2638          * Wait for the response to a credit flush if one's
2639          * pending.
2640          */
2641         while (eq->flags & EQ_CRFLUSHED)
2642                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2643         EQ_UNLOCK(eq);
2644
2645         callout_drain(&eq->tx_callout); /* XXX: iffy */
2646         pause("callout", 10);           /* Still iffy */
2647
2648         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2649 }
2650
2651 static void
2652 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
2653 {
2654         (void) sc;      /* unused */
2655
2656         /* Synchronize with the interrupt handler */
2657         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2658                 pause("iqfree", 1);
2659 }
2660
2661 static void
2662 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
2663 {
2664         mtx_lock(&sc->sfl_lock);
2665         FL_LOCK(fl);
2666         fl->flags |= FL_DOOMED;
2667         FL_UNLOCK(fl);
2668         mtx_unlock(&sc->sfl_lock);
2669
2670         callout_drain(&sc->sfl_callout);
2671         KASSERT((fl->flags & FL_STARVING) == 0,
2672             ("%s: still starving", __func__));
2673 }
2674
2675 static int
2676 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2677     driver_intr_t *handler, void *arg, char *name)
2678 {
2679         int rc;
2680
2681         irq->rid = rid;
2682         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2683             RF_SHAREABLE | RF_ACTIVE);
2684         if (irq->res == NULL) {
2685                 device_printf(sc->dev,
2686                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2687                 return (ENOMEM);
2688         }
2689
2690         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2691             NULL, handler, arg, &irq->tag);
2692         if (rc != 0) {
2693                 device_printf(sc->dev,
2694                     "failed to setup interrupt for rid %d, name %s: %d\n",
2695                     rid, name, rc);
2696         } else if (name)
2697                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2698
2699         return (rc);
2700 }
2701
2702 static int
2703 t4_free_irq(struct adapter *sc, struct irq *irq)
2704 {
2705         if (irq->tag)
2706                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
2707         if (irq->res)
2708                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2709
2710         bzero(irq, sizeof(*irq));
2711
2712         return (0);
2713 }
2714
2715 static void
2716 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2717     unsigned int end)
2718 {
2719         uint32_t *p = (uint32_t *)(buf + start);
2720
2721         for ( ; start <= end; start += sizeof(uint32_t))
2722                 *p++ = t4_read_reg(sc, start);
2723 }
2724
2725 static void
2726 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2727 {
2728         int i;
2729         static const unsigned int reg_ranges[] = {
2730                 0x1008, 0x1108,
2731                 0x1180, 0x11b4,
2732                 0x11fc, 0x123c,
2733                 0x1300, 0x173c,
2734                 0x1800, 0x18fc,
2735                 0x3000, 0x30d8,
2736                 0x30e0, 0x5924,
2737                 0x5960, 0x59d4,
2738                 0x5a00, 0x5af8,
2739                 0x6000, 0x6098,
2740                 0x6100, 0x6150,
2741                 0x6200, 0x6208,
2742                 0x6240, 0x6248,
2743                 0x6280, 0x6338,
2744                 0x6370, 0x638c,
2745                 0x6400, 0x643c,
2746                 0x6500, 0x6524,
2747                 0x6a00, 0x6a38,
2748                 0x6a60, 0x6a78,
2749                 0x6b00, 0x6b84,
2750                 0x6bf0, 0x6c84,
2751                 0x6cf0, 0x6d84,
2752                 0x6df0, 0x6e84,
2753                 0x6ef0, 0x6f84,
2754                 0x6ff0, 0x7084,
2755                 0x70f0, 0x7184,
2756                 0x71f0, 0x7284,
2757                 0x72f0, 0x7384,
2758                 0x73f0, 0x7450,
2759                 0x7500, 0x7530,
2760                 0x7600, 0x761c,
2761                 0x7680, 0x76cc,
2762                 0x7700, 0x7798,
2763                 0x77c0, 0x77fc,
2764                 0x7900, 0x79fc,
2765                 0x7b00, 0x7c38,
2766                 0x7d00, 0x7efc,
2767                 0x8dc0, 0x8e1c,
2768                 0x8e30, 0x8e78,
2769                 0x8ea0, 0x8f6c,
2770                 0x8fc0, 0x9074,
2771                 0x90fc, 0x90fc,
2772                 0x9400, 0x9458,
2773                 0x9600, 0x96bc,
2774                 0x9800, 0x9808,
2775                 0x9820, 0x983c,
2776                 0x9850, 0x9864,
2777                 0x9c00, 0x9c6c,
2778                 0x9c80, 0x9cec,
2779                 0x9d00, 0x9d6c,
2780                 0x9d80, 0x9dec,
2781                 0x9e00, 0x9e6c,
2782                 0x9e80, 0x9eec,
2783                 0x9f00, 0x9f6c,
2784                 0x9f80, 0x9fec,
2785                 0xd004, 0xd03c,
2786                 0xdfc0, 0xdfe0,
2787                 0xe000, 0xea7c,
2788                 0xf000, 0x11190,
2789                 0x19040, 0x1906c,
2790                 0x19078, 0x19080,
2791                 0x1908c, 0x19124,
2792                 0x19150, 0x191b0,
2793                 0x191d0, 0x191e8,
2794                 0x19238, 0x1924c,
2795                 0x193f8, 0x19474,
2796                 0x19490, 0x194f8,
2797                 0x19800, 0x19f30,
2798                 0x1a000, 0x1a06c,
2799                 0x1a0b0, 0x1a120,
2800                 0x1a128, 0x1a138,
2801                 0x1a190, 0x1a1c4,
2802                 0x1a1fc, 0x1a1fc,
2803                 0x1e040, 0x1e04c,
2804                 0x1e284, 0x1e28c,
2805                 0x1e2c0, 0x1e2c0,
2806                 0x1e2e0, 0x1e2e0,
2807                 0x1e300, 0x1e384,
2808                 0x1e3c0, 0x1e3c8,
2809                 0x1e440, 0x1e44c,
2810                 0x1e684, 0x1e68c,
2811                 0x1e6c0, 0x1e6c0,
2812                 0x1e6e0, 0x1e6e0,
2813                 0x1e700, 0x1e784,
2814                 0x1e7c0, 0x1e7c8,
2815                 0x1e840, 0x1e84c,
2816                 0x1ea84, 0x1ea8c,
2817                 0x1eac0, 0x1eac0,
2818                 0x1eae0, 0x1eae0,
2819                 0x1eb00, 0x1eb84,
2820                 0x1ebc0, 0x1ebc8,
2821                 0x1ec40, 0x1ec4c,
2822                 0x1ee84, 0x1ee8c,
2823                 0x1eec0, 0x1eec0,
2824                 0x1eee0, 0x1eee0,
2825                 0x1ef00, 0x1ef84,
2826                 0x1efc0, 0x1efc8,
2827                 0x1f040, 0x1f04c,
2828                 0x1f284, 0x1f28c,
2829                 0x1f2c0, 0x1f2c0,
2830                 0x1f2e0, 0x1f2e0,
2831                 0x1f300, 0x1f384,
2832                 0x1f3c0, 0x1f3c8,
2833                 0x1f440, 0x1f44c,
2834                 0x1f684, 0x1f68c,
2835                 0x1f6c0, 0x1f6c0,
2836                 0x1f6e0, 0x1f6e0,
2837                 0x1f700, 0x1f784,
2838                 0x1f7c0, 0x1f7c8,
2839                 0x1f840, 0x1f84c,
2840                 0x1fa84, 0x1fa8c,
2841                 0x1fac0, 0x1fac0,
2842                 0x1fae0, 0x1fae0,
2843                 0x1fb00, 0x1fb84,
2844                 0x1fbc0, 0x1fbc8,
2845                 0x1fc40, 0x1fc4c,
2846                 0x1fe84, 0x1fe8c,
2847                 0x1fec0, 0x1fec0,
2848                 0x1fee0, 0x1fee0,
2849                 0x1ff00, 0x1ff84,
2850                 0x1ffc0, 0x1ffc8,
2851                 0x20000, 0x2002c,
2852                 0x20100, 0x2013c,
2853                 0x20190, 0x201c8,
2854                 0x20200, 0x20318,
2855                 0x20400, 0x20528,
2856                 0x20540, 0x20614,
2857                 0x21000, 0x21040,
2858                 0x2104c, 0x21060,
2859                 0x210c0, 0x210ec,
2860                 0x21200, 0x21268,
2861                 0x21270, 0x21284,
2862                 0x212fc, 0x21388,
2863                 0x21400, 0x21404,
2864                 0x21500, 0x21518,
2865                 0x2152c, 0x2153c,
2866                 0x21550, 0x21554,
2867                 0x21600, 0x21600,
2868                 0x21608, 0x21628,
2869                 0x21630, 0x2163c,
2870                 0x21700, 0x2171c,
2871                 0x21780, 0x2178c,
2872                 0x21800, 0x21c38,
2873                 0x21c80, 0x21d7c,
2874                 0x21e00, 0x21e04,
2875                 0x22000, 0x2202c,
2876                 0x22100, 0x2213c,
2877                 0x22190, 0x221c8,
2878                 0x22200, 0x22318,
2879                 0x22400, 0x22528,
2880                 0x22540, 0x22614,
2881                 0x23000, 0x23040,
2882                 0x2304c, 0x23060,
2883                 0x230c0, 0x230ec,
2884                 0x23200, 0x23268,
2885                 0x23270, 0x23284,
2886                 0x232fc, 0x23388,
2887                 0x23400, 0x23404,
2888                 0x23500, 0x23518,
2889                 0x2352c, 0x2353c,
2890                 0x23550, 0x23554,
2891                 0x23600, 0x23600,
2892                 0x23608, 0x23628,
2893                 0x23630, 0x2363c,
2894                 0x23700, 0x2371c,
2895                 0x23780, 0x2378c,
2896                 0x23800, 0x23c38,
2897                 0x23c80, 0x23d7c,
2898                 0x23e00, 0x23e04,
2899                 0x24000, 0x2402c,
2900                 0x24100, 0x2413c,
2901                 0x24190, 0x241c8,
2902                 0x24200, 0x24318,
2903                 0x24400, 0x24528,
2904                 0x24540, 0x24614,
2905                 0x25000, 0x25040,
2906                 0x2504c, 0x25060,
2907                 0x250c0, 0x250ec,
2908                 0x25200, 0x25268,
2909                 0x25270, 0x25284,
2910                 0x252fc, 0x25388,
2911                 0x25400, 0x25404,
2912                 0x25500, 0x25518,
2913                 0x2552c, 0x2553c,
2914                 0x25550, 0x25554,
2915                 0x25600, 0x25600,
2916                 0x25608, 0x25628,
2917                 0x25630, 0x2563c,
2918                 0x25700, 0x2571c,
2919                 0x25780, 0x2578c,
2920                 0x25800, 0x25c38,
2921                 0x25c80, 0x25d7c,
2922                 0x25e00, 0x25e04,
2923                 0x26000, 0x2602c,
2924                 0x26100, 0x2613c,
2925                 0x26190, 0x261c8,
2926                 0x26200, 0x26318,
2927                 0x26400, 0x26528,
2928                 0x26540, 0x26614,
2929                 0x27000, 0x27040,
2930                 0x2704c, 0x27060,
2931                 0x270c0, 0x270ec,
2932                 0x27200, 0x27268,
2933                 0x27270, 0x27284,
2934                 0x272fc, 0x27388,
2935                 0x27400, 0x27404,
2936                 0x27500, 0x27518,
2937                 0x2752c, 0x2753c,
2938                 0x27550, 0x27554,
2939                 0x27600, 0x27600,
2940                 0x27608, 0x27628,
2941                 0x27630, 0x2763c,
2942                 0x27700, 0x2771c,
2943                 0x27780, 0x2778c,
2944                 0x27800, 0x27c38,
2945                 0x27c80, 0x27d7c,
2946                 0x27e00, 0x27e04
2947         };
2948
2949         regs->version = 4 | (sc->params.rev << 10);
2950         for (i = 0; i < nitems(reg_ranges); i += 2)
2951                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
2952 }
2953
2954 static void
2955 cxgbe_tick(void *arg)
2956 {
2957         struct port_info *pi = arg;
2958         struct ifnet *ifp = pi->ifp;
2959         struct sge_txq *txq;
2960         int i, drops;
2961         struct port_stats *s = &pi->stats;
2962
2963         PORT_LOCK(pi);
2964         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2965                 PORT_UNLOCK(pi);
2966                 return; /* without scheduling another callout */
2967         }
2968
2969         t4_get_port_stats(pi->adapter, pi->tx_chan, s);
2970
2971         ifp->if_opackets = s->tx_frames - s->tx_pause;
2972         ifp->if_ipackets = s->rx_frames - s->rx_pause;
2973         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
2974         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
2975         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
2976         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
2977         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2978             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
2979             s->rx_trunc3;
2980
2981         drops = s->tx_drop;
2982         for_each_txq(pi, i, txq)
2983                 drops += txq->br->br_drops;
2984         ifp->if_snd.ifq_drops = drops;
2985
2986         ifp->if_oerrors = s->tx_error_frames;
2987         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
2988             s->rx_fcs_err + s->rx_len_err;
2989
2990         callout_schedule(&pi->tick, hz);
2991         PORT_UNLOCK(pi);
2992 }
2993
2994 static void
2995 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
2996 {
2997         struct ifnet *vlan;
2998
2999         if (arg != ifp)
3000                 return;
3001
3002         vlan = VLAN_DEVAT(ifp, vid);
3003         VLAN_SETCOOKIE(vlan, ifp);
3004 }
3005
3006 static int
3007 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
3008 {
3009
3010 #ifdef INVARIANTS
3011         panic("%s: opcode 0x%02x on iq %p with payload %p",
3012             __func__, rss->opcode, iq, m);
3013 #else
3014         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
3015             __func__, rss->opcode, iq, m);
3016         m_freem(m);
3017 #endif
3018         return (EDOOFUS);
3019 }
3020
3021 int
3022 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3023 {
3024         uintptr_t *loc, new;
3025
3026         if (opcode >= nitems(sc->cpl_handler))
3027                 return (EINVAL);
3028
3029         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3030         loc = (uintptr_t *) &sc->cpl_handler[opcode];
3031         atomic_store_rel_ptr(loc, new);
3032
3033         return (0);
3034 }
3035
3036 static int
3037 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
3038 {
3039
3040 #ifdef INVARIANTS
3041         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
3042 #else
3043         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
3044             __func__, iq, ctrl);
3045 #endif
3046         return (EDOOFUS);
3047 }
3048
3049 int
3050 t4_register_an_handler(struct adapter *sc, an_handler_t h)
3051 {
3052         uintptr_t *loc, new;
3053
3054         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
3055         loc = (uintptr_t *) &sc->an_handler;
3056         atomic_store_rel_ptr(loc, new);
3057
3058         return (0);
3059 }
3060
3061 static int
3062 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
3063 {
3064         __be64 *r = __DECONST(__be64 *, rpl);
3065         struct cpl_fw6_msg *cpl = member2struct(cpl_fw6_msg, data, r);
3066
3067 #ifdef INVARIANTS
3068         panic("%s: fw_msg type %d", __func__, cpl->type);
3069 #else
3070         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
3071 #endif
3072         return (EDOOFUS);
3073 }
3074
3075 int
3076 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
3077 {
3078         uintptr_t *loc, new;
3079
3080         if (type >= nitems(sc->fw_msg_handler))
3081                 return (EINVAL);
3082
3083         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
3084         loc = (uintptr_t *) &sc->fw_msg_handler[type];
3085         atomic_store_rel_ptr(loc, new);
3086
3087         return (0);
3088 }
3089
3090 static int
3091 t4_sysctls(struct adapter *sc)
3092 {
3093         struct sysctl_ctx_list *ctx;
3094         struct sysctl_oid *oid;
3095         struct sysctl_oid_list *children, *c0;
3096         static char *caps[] = {
3097                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
3098                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",       /* caps[1] niccaps */
3099                 "\20\1TOE",                             /* caps[2] toecaps */
3100                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
3101                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
3102                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
3103                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
3104                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
3105         };
3106
3107         ctx = device_get_sysctl_ctx(sc->dev);
3108
3109         /*
3110          * dev.t4nex.X.
3111          */
3112         oid = device_get_sysctl_tree(sc->dev);
3113         c0 = children = SYSCTL_CHILDREN(oid);
3114
3115         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3116             &sc->params.nports, 0, "# of ports");
3117
3118         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3119             &sc->params.rev, 0, "chip hardware revision");
3120
3121         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3122             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3123
3124         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3125             CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
3126
3127         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3128             &sc->cfcsum, 0, "config file checksum");
3129
3130         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3131             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3132             sysctl_bitfield, "A", "available link capabilities");
3133
3134         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3135             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3136             sysctl_bitfield, "A", "available NIC capabilities");
3137
3138         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3139             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3140             sysctl_bitfield, "A", "available TCP offload capabilities");
3141
3142         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3143             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3144             sysctl_bitfield, "A", "available RDMA capabilities");
3145
3146         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3147             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3148             sysctl_bitfield, "A", "available iSCSI capabilities");
3149
3150         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3151             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3152             sysctl_bitfield, "A", "available FCoE capabilities");
3153
3154         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3155             &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3156
3157         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3158             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3159             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3160             "interrupt holdoff timer values (us)");
3161
3162         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3163             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3164             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3165             "interrupt holdoff packet counter values");
3166
3167 #ifdef SBUF_DRAIN
3168         /*
3169          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3170          */
3171         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3172             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3173             "logs and miscellaneous information");
3174         children = SYSCTL_CHILDREN(oid);
3175
3176         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3177             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3178             sysctl_cctrl, "A", "congestion control");
3179
3180         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3181             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3182             sysctl_cpl_stats, "A", "CPL statistics");
3183
3184         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3185             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3186             sysctl_ddp_stats, "A", "DDP statistics");
3187
3188         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3189             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3190             sysctl_devlog, "A", "firmware's device log");
3191
3192         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3193             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3194             sysctl_fcoe_stats, "A", "FCoE statistics");
3195
3196         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3197             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3198             sysctl_hw_sched, "A", "hardware scheduler ");
3199
3200         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3201             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3202             sysctl_l2t, "A", "hardware L2 table");
3203
3204         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3205             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3206             sysctl_lb_stats, "A", "loopback statistics");
3207
3208         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3209             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3210             sysctl_meminfo, "A", "memory regions");
3211
3212         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3213             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3214             sysctl_path_mtus, "A", "path MTUs");
3215
3216         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3217             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3218             sysctl_pm_stats, "A", "PM statistics");
3219
3220         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3221             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3222             sysctl_rdma_stats, "A", "RDMA statistics");
3223
3224         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3225             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3226             sysctl_tcp_stats, "A", "TCP statistics");
3227
3228         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3229             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3230             sysctl_tids, "A", "TID information");
3231
3232         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3233             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3234             sysctl_tp_err_stats, "A", "TP error statistics");
3235
3236         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3237             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3238             sysctl_tx_rate, "A", "Tx rate");
3239 #endif
3240
3241 #ifdef TCP_OFFLOAD
3242         if (is_offload(sc)) {
3243                 /*
3244                  * dev.t4nex.X.toe.
3245                  */
3246                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3247                     NULL, "TOE parameters");
3248                 children = SYSCTL_CHILDREN(oid);
3249
3250                 sc->tt.sndbuf = 256 * 1024;
3251                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3252                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
3253
3254                 sc->tt.ddp = 0;
3255                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3256                     &sc->tt.ddp, 0, "DDP allowed");
3257
3258                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
3259                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3260                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
3261
3262                 sc->tt.ddp_thres =
3263                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
3264                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3265                     &sc->tt.ddp_thres, 0, "DDP threshold");
3266         }
3267 #endif
3268
3269
3270         return (0);
3271 }
3272
3273 static int
3274 cxgbe_sysctls(struct port_info *pi)
3275 {
3276         struct sysctl_ctx_list *ctx;
3277         struct sysctl_oid *oid;
3278         struct sysctl_oid_list *children;
3279
3280         ctx = device_get_sysctl_ctx(pi->dev);
3281
3282         /*
3283          * dev.cxgbe.X.
3284          */
3285         oid = device_get_sysctl_tree(pi->dev);
3286         children = SYSCTL_CHILDREN(oid);
3287
3288         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3289             &pi->nrxq, 0, "# of rx queues");
3290         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3291             &pi->ntxq, 0, "# of tx queues");
3292         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3293             &pi->first_rxq, 0, "index of first rx queue");
3294         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3295             &pi->first_txq, 0, "index of first tx queue");
3296
3297 #ifdef TCP_OFFLOAD
3298         if (is_offload(pi->adapter)) {
3299                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3300                     &pi->nofldrxq, 0,
3301                     "# of rx queues for offloaded TCP connections");
3302                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3303                     &pi->nofldtxq, 0,
3304                     "# of tx queues for offloaded TCP connections");
3305                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3306                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3307                     "index of first TOE rx queue");
3308                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3309                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
3310                     "index of first TOE tx queue");
3311         }
3312 #endif
3313
3314         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3315             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3316             "holdoff timer index");
3317         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3318             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3319             "holdoff packet counter index");
3320
3321         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3322             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3323             "rx queue size");
3324         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3325             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3326             "tx queue size");
3327
3328         /*
3329          * dev.cxgbe.X.stats.
3330          */
3331         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3332             NULL, "port statistics");
3333         children = SYSCTL_CHILDREN(oid);
3334
3335 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3336         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3337             CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3338             sysctl_handle_t4_reg64, "QU", desc)
3339
3340         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3341             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3342         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3343             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3344         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3345             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3346         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3347             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3348         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3349             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3350         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3351             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3352         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3353             "# of tx frames in this range",
3354             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3355         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3356             "# of tx frames in this range",
3357             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3358         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3359             "# of tx frames in this range",
3360             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3361         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3362             "# of tx frames in this range",
3363             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3364         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3365             "# of tx frames in this range",
3366             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3367         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3368             "# of tx frames in this range",
3369             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3370         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3371             "# of tx frames in this range",
3372             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3373         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3374             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3375         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3376             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3377         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3378             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3379         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3380             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3381         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3382             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3383         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3384             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3385         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3386             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3387         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3388             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3389         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3390             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3391         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3392             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3393
3394         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3395             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3396         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3397             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3398         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3399             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3400         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3401             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3402         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3403             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3404         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3405             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3406         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3407             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3408         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3409             "# of frames received with bad FCS",
3410             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3411         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3412             "# of frames received with length error",
3413             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3414         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3415             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3416         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3417             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3418         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3419             "# of rx frames in this range",
3420             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3421         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3422             "# of rx frames in this range",
3423             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3424         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3425             "# of rx frames in this range",
3426             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3427         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3428             "# of rx frames in this range",
3429             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3430         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3431             "# of rx frames in this range",
3432             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3433         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3434             "# of rx frames in this range",
3435             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3436         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3437             "# of rx frames in this range",
3438             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3439         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3440             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3441         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3442             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3443         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3444             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3445         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3446             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3447         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3448             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3449         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3450             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3451         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3452             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3453         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3454             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3455         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3456             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3457
3458 #undef SYSCTL_ADD_T4_REG64
3459
3460 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3461         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3462             &pi->stats.name, desc)
3463
3464         /* We get these from port_stats and they may be stale by upto 1s */
3465         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3466             "# drops due to buffer-group 0 overflows");
3467         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3468             "# drops due to buffer-group 1 overflows");
3469         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3470             "# drops due to buffer-group 2 overflows");
3471         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3472             "# drops due to buffer-group 3 overflows");
3473         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3474             "# of buffer-group 0 truncated packets");
3475         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3476             "# of buffer-group 1 truncated packets");
3477         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3478             "# of buffer-group 2 truncated packets");
3479         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3480             "# of buffer-group 3 truncated packets");
3481
3482 #undef SYSCTL_ADD_T4_PORTSTAT
3483
3484         return (0);
3485 }
3486
3487 static int
3488 sysctl_int_array(SYSCTL_HANDLER_ARGS)
3489 {
3490         int rc, *i;
3491         struct sbuf sb;
3492
3493         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3494         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3495                 sbuf_printf(&sb, "%d ", *i);
3496         sbuf_trim(&sb);
3497         sbuf_finish(&sb);
3498         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3499         sbuf_delete(&sb);
3500         return (rc);
3501 }
3502
3503 static int
3504 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
3505 {
3506         int rc;
3507         struct sbuf *sb;
3508
3509         rc = sysctl_wire_old_buffer(req, 0);
3510         if (rc != 0)
3511                 return(rc);
3512
3513         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3514         if (sb == NULL)
3515                 return (ENOMEM);
3516
3517         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3518         rc = sbuf_finish(sb);
3519         sbuf_delete(sb);
3520
3521         return (rc);
3522 }
3523
3524 static int
3525 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3526 {
3527         struct port_info *pi = arg1;
3528         struct adapter *sc = pi->adapter;
3529         int idx, rc, i;
3530
3531         idx = pi->tmr_idx;
3532
3533         rc = sysctl_handle_int(oidp, &idx, 0, req);
3534         if (rc != 0 || req->newptr == NULL)
3535                 return (rc);
3536
3537         if (idx < 0 || idx >= SGE_NTIMERS)
3538                 return (EINVAL);
3539
3540         ADAPTER_LOCK(sc);
3541         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3542         if (rc == 0) {
3543                 struct sge_rxq *rxq;
3544                 uint8_t v;
3545
3546                 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3547                 for_each_rxq(pi, i, rxq) {
3548 #ifdef atomic_store_rel_8
3549                         atomic_store_rel_8(&rxq->iq.intr_params, v);
3550 #else
3551                         rxq->iq.intr_params = v;
3552 #endif
3553                 }
3554                 pi->tmr_idx = idx;
3555         }
3556
3557         ADAPTER_UNLOCK(sc);
3558         return (rc);
3559 }
3560
3561 static int
3562 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3563 {
3564         struct port_info *pi = arg1;
3565         struct adapter *sc = pi->adapter;
3566         int idx, rc;
3567
3568         idx = pi->pktc_idx;
3569
3570         rc = sysctl_handle_int(oidp, &idx, 0, req);
3571         if (rc != 0 || req->newptr == NULL)
3572                 return (rc);
3573
3574         if (idx < -1 || idx >= SGE_NCOUNTERS)
3575                 return (EINVAL);
3576
3577         ADAPTER_LOCK(sc);
3578         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3579         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3580                 rc = EBUSY; /* cannot be changed once the queues are created */
3581
3582         if (rc == 0)
3583                 pi->pktc_idx = idx;
3584
3585         ADAPTER_UNLOCK(sc);
3586         return (rc);
3587 }
3588
3589 static int
3590 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3591 {
3592         struct port_info *pi = arg1;
3593         struct adapter *sc = pi->adapter;
3594         int qsize, rc;
3595
3596         qsize = pi->qsize_rxq;
3597
3598         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3599         if (rc != 0 || req->newptr == NULL)
3600                 return (rc);
3601
3602         if (qsize < 128 || (qsize & 7))
3603                 return (EINVAL);
3604
3605         ADAPTER_LOCK(sc);
3606         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3607         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3608                 rc = EBUSY; /* cannot be changed once the queues are created */
3609
3610         if (rc == 0)
3611                 pi->qsize_rxq = qsize;
3612
3613         ADAPTER_UNLOCK(sc);
3614         return (rc);
3615 }
3616
3617 static int
3618 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3619 {
3620         struct port_info *pi = arg1;
3621         struct adapter *sc = pi->adapter;
3622         int qsize, rc;
3623
3624         qsize = pi->qsize_txq;
3625
3626         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3627         if (rc != 0 || req->newptr == NULL)
3628                 return (rc);
3629
3630         if (qsize < 128)
3631                 return (EINVAL);
3632
3633         ADAPTER_LOCK(sc);
3634         rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3635         if (rc == 0 && pi->flags & PORT_INIT_DONE)
3636                 rc = EBUSY; /* cannot be changed once the queues are created */
3637
3638         if (rc == 0)
3639                 pi->qsize_txq = qsize;
3640
3641         ADAPTER_UNLOCK(sc);
3642         return (rc);
3643 }
3644
3645 static int
3646 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3647 {
3648         struct adapter *sc = arg1;
3649         int reg = arg2;
3650         uint64_t val;
3651
3652         val = t4_read_reg64(sc, reg);
3653
3654         return (sysctl_handle_64(oidp, &val, 0, req));
3655 }
3656
3657 #ifdef SBUF_DRAIN
3658 static int
3659 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
3660 {
3661         struct adapter *sc = arg1;
3662         struct sbuf *sb;
3663         int rc, i;
3664         uint16_t incr[NMTUS][NCCTRL_WIN];
3665         static const char *dec_fac[] = {
3666                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3667                 "0.9375"
3668         };
3669
3670         rc = sysctl_wire_old_buffer(req, 0);
3671         if (rc != 0)
3672                 return (rc);
3673
3674         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3675         if (sb == NULL)
3676                 return (ENOMEM);
3677
3678         t4_read_cong_tbl(sc, incr);
3679
3680         for (i = 0; i < NCCTRL_WIN; ++i) {
3681                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3682                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3683                     incr[5][i], incr[6][i], incr[7][i]);
3684                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3685                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3686                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3687                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3688         }
3689
3690         rc = sbuf_finish(sb);
3691         sbuf_delete(sb);
3692
3693         return (rc);
3694 }
3695
3696 static int
3697 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
3698 {
3699         struct adapter *sc = arg1;
3700         struct sbuf *sb;
3701         int rc;
3702         struct tp_cpl_stats stats;
3703
3704         rc = sysctl_wire_old_buffer(req, 0);
3705         if (rc != 0)
3706                 return (rc);
3707
3708         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3709         if (sb == NULL)
3710                 return (ENOMEM);
3711
3712         t4_tp_get_cpl_stats(sc, &stats);
3713
3714         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
3715             "channel 3\n");
3716         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
3717                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
3718         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
3719                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
3720
3721         rc = sbuf_finish(sb);
3722         sbuf_delete(sb);
3723
3724         return (rc);
3725 }
3726
3727 static int
3728 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
3729 {
3730         struct adapter *sc = arg1;
3731         struct sbuf *sb;
3732         int rc;
3733         struct tp_usm_stats stats;
3734
3735         rc = sysctl_wire_old_buffer(req, 0);
3736         if (rc != 0)
3737                 return(rc);
3738
3739         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3740         if (sb == NULL)
3741                 return (ENOMEM);
3742
3743         t4_get_usm_stats(sc, &stats);
3744
3745         sbuf_printf(sb, "Frames: %u\n", stats.frames);
3746         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
3747         sbuf_printf(sb, "Drops:  %u", stats.drops);
3748
3749         rc = sbuf_finish(sb);
3750         sbuf_delete(sb);
3751
3752         return (rc);
3753 }
3754
3755 const char *devlog_level_strings[] = {
3756         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
3757         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
3758         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
3759         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
3760         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
3761         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
3762 };
3763
3764 const char *devlog_facility_strings[] = {
3765         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
3766         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
3767         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
3768         [FW_DEVLOG_FACILITY_RES]        = "RES",
3769         [FW_DEVLOG_FACILITY_HW]         = "HW",
3770         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
3771         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
3772         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
3773         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
3774         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
3775         [FW_DEVLOG_FACILITY_VI]         = "VI",
3776         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
3777         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
3778         [FW_DEVLOG_FACILITY_TM]         = "TM",
3779         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
3780         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
3781         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
3782         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
3783         [FW_DEVLOG_FACILITY_RI]         = "RI",
3784         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
3785         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
3786         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
3787         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
3788 };
3789
3790 static int
3791 sysctl_devlog(SYSCTL_HANDLER_ARGS)
3792 {
3793         struct adapter *sc = arg1;
3794         struct devlog_params *dparams = &sc->params.devlog;
3795         struct fw_devlog_e *buf, *e;
3796         int i, j, rc, nentries, first = 0;
3797         struct sbuf *sb;
3798         uint64_t ftstamp = UINT64_MAX;
3799
3800         if (dparams->start == 0)
3801                 return (ENXIO);
3802
3803         nentries = dparams->size / sizeof(struct fw_devlog_e);
3804
3805         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
3806         if (buf == NULL)
3807                 return (ENOMEM);
3808
3809         rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
3810             (void *)buf);
3811         if (rc != 0)
3812                 goto done;
3813
3814         for (i = 0; i < nentries; i++) {
3815                 e = &buf[i];
3816
3817                 if (e->timestamp == 0)
3818                         break;  /* end */
3819
3820                 e->timestamp = be64toh(e->timestamp);
3821                 e->seqno = be32toh(e->seqno);
3822                 for (j = 0; j < 8; j++)
3823                         e->params[j] = be32toh(e->params[j]);
3824
3825                 if (e->timestamp < ftstamp) {
3826                         ftstamp = e->timestamp;
3827                         first = i;
3828                 }
3829         }
3830
3831         if (buf[first].timestamp == 0)
3832                 goto done;      /* nothing in the log */
3833
3834         rc = sysctl_wire_old_buffer(req, 0);
3835         if (rc != 0)
3836                 goto done;
3837
3838         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3839         if (sb == NULL) {
3840                 rc = ENOMEM;
3841                 goto done;
3842         }
3843         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
3844             "Seq#", "Tstamp", "Level", "Facility", "Message");
3845
3846         i = first;
3847         do {
3848                 e = &buf[i];
3849                 if (e->timestamp == 0)
3850                         break;  /* end */
3851
3852                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
3853                     e->seqno, e->timestamp,
3854                     (e->level < nitems(devlog_level_strings) ?
3855                         devlog_level_strings[e->level] : "UNKNOWN"),
3856                     (e->facility < nitems(devlog_facility_strings) ?
3857                         devlog_facility_strings[e->facility] : "UNKNOWN"));
3858                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
3859                     e->params[2], e->params[3], e->params[4],
3860                     e->params[5], e->params[6], e->params[7]);
3861
3862                 if (++i == nentries)
3863                         i = 0;
3864         } while (i != first);
3865
3866         rc = sbuf_finish(sb);
3867         sbuf_delete(sb);
3868 done:
3869         free(buf, M_CXGBE);
3870         return (rc);
3871 }
3872
3873 static int
3874 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
3875 {
3876         struct adapter *sc = arg1;
3877         struct sbuf *sb;
3878         int rc;
3879         struct tp_fcoe_stats stats[4];
3880
3881         rc = sysctl_wire_old_buffer(req, 0);
3882         if (rc != 0)
3883                 return (rc);
3884
3885         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3886         if (sb == NULL)
3887                 return (ENOMEM);
3888
3889         t4_get_fcoe_stats(sc, 0, &stats[0]);
3890         t4_get_fcoe_stats(sc, 1, &stats[1]);
3891         t4_get_fcoe_stats(sc, 2, &stats[2]);
3892         t4_get_fcoe_stats(sc, 3, &stats[3]);
3893
3894         sbuf_printf(sb, "                   channel 0        channel 1        "
3895             "channel 2        channel 3\n");
3896         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
3897             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
3898             stats[3].octetsDDP);
3899         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
3900             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
3901         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
3902             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
3903             stats[3].framesDrop);
3904
3905         rc = sbuf_finish(sb);
3906         sbuf_delete(sb);
3907
3908         return (rc);
3909 }
3910
3911 static int
3912 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
3913 {
3914         struct adapter *sc = arg1;
3915         struct sbuf *sb;
3916         int rc, i;
3917         unsigned int map, kbps, ipg, mode;
3918         unsigned int pace_tab[NTX_SCHED];
3919
3920         rc = sysctl_wire_old_buffer(req, 0);
3921         if (rc != 0)
3922                 return (rc);
3923
3924         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3925         if (sb == NULL)
3926                 return (ENOMEM);
3927
3928         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
3929         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
3930         t4_read_pace_tbl(sc, pace_tab);
3931
3932         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
3933             "Class IPG (0.1 ns)   Flow IPG (us)");
3934
3935         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
3936                 t4_get_tx_sched(sc, i, &kbps, &ipg);
3937                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
3938                     (mode & (1 << i)) ? "flow" : "class", map & 3);
3939                 if (kbps)
3940                         sbuf_printf(sb, "%9u     ", kbps);
3941                 else
3942                         sbuf_printf(sb, " disabled     ");
3943
3944                 if (ipg)
3945                         sbuf_printf(sb, "%13u        ", ipg);
3946                 else
3947                         sbuf_printf(sb, "     disabled        ");
3948
3949                 if (pace_tab[i])
3950                         sbuf_printf(sb, "%10u", pace_tab[i]);
3951                 else
3952                         sbuf_printf(sb, "  disabled");
3953         }
3954
3955         rc = sbuf_finish(sb);
3956         sbuf_delete(sb);
3957
3958         return (rc);
3959 }
3960
3961 static int
3962 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
3963 {
3964         struct adapter *sc = arg1;
3965         struct sbuf *sb;
3966         int rc, i, j;
3967         uint64_t *p0, *p1;
3968         struct lb_port_stats s[2];
3969         static const char *stat_name[] = {
3970                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
3971                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
3972                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
3973                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
3974                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
3975                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
3976                 "BG2FramesTrunc:", "BG3FramesTrunc:"
3977         };
3978
3979         rc = sysctl_wire_old_buffer(req, 0);
3980         if (rc != 0)
3981                 return (rc);
3982
3983         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3984         if (sb == NULL)
3985                 return (ENOMEM);
3986
3987         memset(s, 0, sizeof(s));
3988
3989         for (i = 0; i < 4; i += 2) {
3990                 t4_get_lb_stats(sc, i, &s[0]);
3991                 t4_get_lb_stats(sc, i + 1, &s[1]);
3992
3993                 p0 = &s[0].octets;
3994                 p1 = &s[1].octets;
3995                 sbuf_printf(sb, "%s                       Loopback %u"
3996                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
3997
3998                 for (j = 0; j < nitems(stat_name); j++)
3999                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
4000                                    *p0++, *p1++);
4001         }
4002
4003         rc = sbuf_finish(sb);
4004         sbuf_delete(sb);
4005
4006         return (rc);
4007 }
4008
4009 struct mem_desc {
4010         unsigned int base;
4011         unsigned int limit;
4012         unsigned int idx;
4013 };
4014
4015 static int
4016 mem_desc_cmp(const void *a, const void *b)
4017 {
4018         return ((const struct mem_desc *)a)->base -
4019                ((const struct mem_desc *)b)->base;
4020 }
4021
4022 static void
4023 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
4024     unsigned int to)
4025 {
4026         unsigned int size;
4027
4028         size = to - from + 1;
4029         if (size == 0)
4030                 return;
4031
4032         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
4033         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
4034 }
4035
4036 static int
4037 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
4038 {
4039         struct adapter *sc = arg1;
4040         struct sbuf *sb;
4041         int rc, i, n;
4042         uint32_t lo, hi;
4043         static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
4044         static const char *region[] = {
4045                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
4046                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
4047                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
4048                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
4049                 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
4050                 "ULPTX state:", "On-chip queues:"
4051         };
4052         struct mem_desc avail[3];
4053         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
4054         struct mem_desc *md = mem;
4055
4056         rc = sysctl_wire_old_buffer(req, 0);
4057         if (rc != 0)
4058                 return (rc);
4059
4060         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4061         if (sb == NULL)
4062                 return (ENOMEM);
4063
4064         for (i = 0; i < nitems(mem); i++) {
4065                 mem[i].limit = 0;
4066                 mem[i].idx = i;
4067         }
4068
4069         /* Find and sort the populated memory ranges */
4070         i = 0;
4071         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4072         if (lo & F_EDRAM0_ENABLE) {
4073                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4074                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
4075                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
4076                 avail[i].idx = 0;
4077                 i++;
4078         }
4079         if (lo & F_EDRAM1_ENABLE) {
4080                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4081                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
4082                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
4083                 avail[i].idx = 1;
4084                 i++;
4085         }
4086         if (lo & F_EXT_MEM_ENABLE) {
4087                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4088                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
4089                 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
4090                 avail[i].idx = 2;
4091                 i++;
4092         }
4093         if (!i)                                    /* no memory available */
4094                 return 0;
4095         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
4096
4097         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
4098         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
4099         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
4100         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4101         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
4102         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
4103         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
4104         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
4105         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
4106
4107         /* the next few have explicit upper bounds */
4108         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
4109         md->limit = md->base - 1 +
4110                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
4111                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
4112         md++;
4113
4114         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4115         md->limit = md->base - 1 +
4116                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4117                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4118         md++;
4119
4120         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4121                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4122                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4123                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4124         } else {
4125                 md->base = 0;
4126                 md->idx = nitems(region);  /* hide it */
4127         }
4128         md++;
4129
4130 #define ulp_region(reg) \
4131         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4132         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4133
4134         ulp_region(RX_ISCSI);
4135         ulp_region(RX_TDDP);
4136         ulp_region(TX_TPT);
4137         ulp_region(RX_STAG);
4138         ulp_region(RX_RQ);
4139         ulp_region(RX_RQUDP);
4140         ulp_region(RX_PBL);
4141         ulp_region(TX_PBL);
4142 #undef ulp_region
4143
4144         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4145         md->limit = md->base + sc->tids.ntids - 1;
4146         md++;
4147         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4148         md->limit = md->base + sc->tids.ntids - 1;
4149         md++;
4150
4151         md->base = sc->vres.ocq.start;
4152         if (sc->vres.ocq.size)
4153                 md->limit = md->base + sc->vres.ocq.size - 1;
4154         else
4155                 md->idx = nitems(region);  /* hide it */
4156         md++;
4157
4158         /* add any address-space holes, there can be up to 3 */
4159         for (n = 0; n < i - 1; n++)
4160                 if (avail[n].limit < avail[n + 1].base)
4161                         (md++)->base = avail[n].limit;
4162         if (avail[n].limit)
4163                 (md++)->base = avail[n].limit;
4164
4165         n = md - mem;
4166         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4167
4168         for (lo = 0; lo < i; lo++)
4169                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4170                                 avail[lo].limit - 1);
4171
4172         sbuf_printf(sb, "\n");
4173         for (i = 0; i < n; i++) {
4174                 if (mem[i].idx >= nitems(region))
4175                         continue;                        /* skip holes */
4176                 if (!mem[i].limit)
4177                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4178                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
4179                                 mem[i].limit);
4180         }
4181
4182         sbuf_printf(sb, "\n");
4183         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4184         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4185         mem_region_show(sb, "uP RAM:", lo, hi);
4186
4187         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4188         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4189         mem_region_show(sb, "uP Extmem2:", lo, hi);
4190
4191         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4192         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4193                    G_PMRXMAXPAGE(lo),
4194                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4195                    (lo & F_PMRXNUMCHN) ? 2 : 1);
4196
4197         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4198         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4199         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4200                    G_PMTXMAXPAGE(lo),
4201                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4202                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4203         sbuf_printf(sb, "%u p-structs\n",
4204                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4205
4206         for (i = 0; i < 4; i++) {
4207                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4208                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4209                            i, G_USED(lo), G_ALLOC(lo));
4210         }
4211         for (i = 0; i < 4; i++) {
4212                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4213                 sbuf_printf(sb,
4214                            "\nLoopback %d using %u pages out of %u allocated",
4215                            i, G_USED(lo), G_ALLOC(lo));
4216         }
4217
4218         rc = sbuf_finish(sb);
4219         sbuf_delete(sb);
4220
4221         return (rc);
4222 }
4223
4224 static int
4225 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4226 {
4227         struct adapter *sc = arg1;
4228         struct sbuf *sb;
4229         int rc;
4230         uint16_t mtus[NMTUS];
4231
4232         rc = sysctl_wire_old_buffer(req, 0);
4233         if (rc != 0)
4234                 return (rc);
4235
4236         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4237         if (sb == NULL)
4238                 return (ENOMEM);
4239
4240         t4_read_mtu_tbl(sc, mtus, NULL);
4241
4242         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4243             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4244             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4245             mtus[14], mtus[15]);
4246
4247         rc = sbuf_finish(sb);
4248         sbuf_delete(sb);
4249
4250         return (rc);
4251 }
4252
4253 static int
4254 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4255 {
4256         struct adapter *sc = arg1;
4257         struct sbuf *sb;
4258         int rc, i;
4259         uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4260         uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4261         static const char *pm_stats[] = {
4262                 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4263         };
4264
4265         rc = sysctl_wire_old_buffer(req, 0);
4266         if (rc != 0)
4267                 return (rc);
4268
4269         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4270         if (sb == NULL)
4271                 return (ENOMEM);
4272
4273         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4274         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4275
4276         sbuf_printf(sb, "                Tx count            Tx cycles    "
4277             "Rx count            Rx cycles");
4278         for (i = 0; i < PM_NSTATS; i++)
4279                 sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4280                     pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4281
4282         rc = sbuf_finish(sb);
4283         sbuf_delete(sb);
4284
4285         return (rc);
4286 }
4287
4288 static int
4289 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4290 {
4291         struct adapter *sc = arg1;
4292         struct sbuf *sb;
4293         int rc;
4294         struct tp_rdma_stats stats;
4295
4296         rc = sysctl_wire_old_buffer(req, 0);
4297         if (rc != 0)
4298                 return (rc);
4299
4300         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4301         if (sb == NULL)
4302                 return (ENOMEM);
4303
4304         t4_tp_get_rdma_stats(sc, &stats);
4305         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4306         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4307
4308         rc = sbuf_finish(sb);
4309         sbuf_delete(sb);
4310
4311         return (rc);
4312 }
4313
4314 static int
4315 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4316 {
4317         struct adapter *sc = arg1;
4318         struct sbuf *sb;
4319         int rc;
4320         struct tp_tcp_stats v4, v6;
4321
4322         rc = sysctl_wire_old_buffer(req, 0);
4323         if (rc != 0)
4324                 return (rc);
4325
4326         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4327         if (sb == NULL)
4328                 return (ENOMEM);
4329
4330         t4_tp_get_tcp_stats(sc, &v4, &v6);
4331         sbuf_printf(sb,
4332             "                                IP                 IPv6\n");
4333         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4334             v4.tcpOutRsts, v6.tcpOutRsts);
4335         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4336             v4.tcpInSegs, v6.tcpInSegs);
4337         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4338             v4.tcpOutSegs, v6.tcpOutSegs);
4339         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4340             v4.tcpRetransSegs, v6.tcpRetransSegs);
4341
4342         rc = sbuf_finish(sb);
4343         sbuf_delete(sb);
4344
4345         return (rc);
4346 }
4347
4348 static int
4349 sysctl_tids(SYSCTL_HANDLER_ARGS)
4350 {
4351         struct adapter *sc = arg1;
4352         struct sbuf *sb;
4353         int rc;
4354         struct tid_info *t = &sc->tids;
4355
4356         rc = sysctl_wire_old_buffer(req, 0);
4357         if (rc != 0)
4358                 return (rc);
4359
4360         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4361         if (sb == NULL)
4362                 return (ENOMEM);
4363
4364         if (t->natids) {
4365                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4366                     t->atids_in_use);
4367         }
4368
4369         if (t->ntids) {
4370                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4371                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4372
4373                         if (b) {
4374                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4375                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4376                                     t->ntids - 1);
4377                         } else {
4378                                 sbuf_printf(sb, "TID range: %u-%u",
4379                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4380                                     t->ntids - 1);
4381                         }
4382                 } else
4383                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4384                 sbuf_printf(sb, ", in use: %u\n",
4385                     atomic_load_acq_int(&t->tids_in_use));
4386         }
4387
4388         if (t->nstids) {
4389                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4390                     t->stid_base + t->nstids - 1, t->stids_in_use);
4391         }
4392
4393         if (t->nftids) {
4394                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4395                     t->ftid_base + t->nftids - 1);
4396         }
4397
4398         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4399             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4400             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4401
4402         rc = sbuf_finish(sb);
4403         sbuf_delete(sb);
4404
4405         return (rc);
4406 }
4407
4408 static int
4409 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4410 {
4411         struct adapter *sc = arg1;
4412         struct sbuf *sb;
4413         int rc;
4414         struct tp_err_stats stats;
4415
4416         rc = sysctl_wire_old_buffer(req, 0);
4417         if (rc != 0)
4418                 return (rc);
4419
4420         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4421         if (sb == NULL)
4422                 return (ENOMEM);
4423
4424         t4_tp_get_err_stats(sc, &stats);
4425
4426         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4427                       "channel 3\n");
4428         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4429             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4430             stats.macInErrs[3]);
4431         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4432             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4433             stats.hdrInErrs[3]);
4434         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4435             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4436             stats.tcpInErrs[3]);
4437         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4438             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4439             stats.tcp6InErrs[3]);
4440         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4441             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4442             stats.tnlCongDrops[3]);
4443         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4444             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4445             stats.tnlTxDrops[3]);
4446         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4447             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4448             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4449         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4450             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4451             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4452         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4453             stats.ofldNoNeigh, stats.ofldCongDefer);
4454
4455         rc = sbuf_finish(sb);
4456         sbuf_delete(sb);
4457
4458         return (rc);
4459 }
4460
4461 static int
4462 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4463 {
4464         struct adapter *sc = arg1;
4465         struct sbuf *sb;
4466         int rc;
4467         u64 nrate[NCHAN], orate[NCHAN];
4468
4469         rc = sysctl_wire_old_buffer(req, 0);
4470         if (rc != 0)
4471                 return (rc);
4472
4473         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4474         if (sb == NULL)
4475                 return (ENOMEM);
4476
4477         t4_get_chan_txrate(sc, nrate, orate);
4478         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4479                  "channel 3\n");
4480         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4481             nrate[0], nrate[1], nrate[2], nrate[3]);
4482         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4483             orate[0], orate[1], orate[2], orate[3]);
4484
4485         rc = sbuf_finish(sb);
4486         sbuf_delete(sb);
4487
4488         return (rc);
4489 }
4490 #endif
4491
4492 static inline void
4493 txq_start(struct ifnet *ifp, struct sge_txq *txq)
4494 {
4495         struct buf_ring *br;
4496         struct mbuf *m;
4497
4498         TXQ_LOCK_ASSERT_OWNED(txq);
4499
4500         br = txq->br;
4501         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4502         if (m)
4503                 t4_eth_tx(ifp, txq, m);
4504 }
4505
4506 void
4507 t4_tx_callout(void *arg)
4508 {
4509         struct sge_eq *eq = arg;
4510         struct adapter *sc;
4511
4512         if (EQ_TRYLOCK(eq) == 0)
4513                 goto reschedule;
4514
4515         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4516                 EQ_UNLOCK(eq);
4517 reschedule:
4518                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
4519                         callout_schedule(&eq->tx_callout, 1);
4520                 return;
4521         }
4522
4523         EQ_LOCK_ASSERT_OWNED(eq);
4524
4525         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4526
4527                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4528                         struct sge_txq *txq = arg;
4529                         struct port_info *pi = txq->ifp->if_softc;
4530
4531                         sc = pi->adapter;
4532                 } else {
4533                         struct sge_wrq *wrq = arg;
4534
4535                         sc = wrq->adapter;
4536                 }
4537
4538                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4539         }
4540
4541         EQ_UNLOCK(eq);
4542 }
4543
4544 void
4545 t4_tx_task(void *arg, int count)
4546 {
4547         struct sge_eq *eq = arg;
4548
4549         EQ_LOCK(eq);
4550         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4551                 struct sge_txq *txq = arg;
4552                 txq_start(txq->ifp, txq);
4553         } else {
4554                 struct sge_wrq *wrq = arg;
4555                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4556         }
4557         EQ_UNLOCK(eq);
4558 }
4559
4560 static uint32_t
4561 fconf_to_mode(uint32_t fconf)
4562 {
4563         uint32_t mode;
4564
4565         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4566             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4567
4568         if (fconf & F_FRAGMENTATION)
4569                 mode |= T4_FILTER_IP_FRAGMENT;
4570
4571         if (fconf & F_MPSHITTYPE)
4572                 mode |= T4_FILTER_MPS_HIT_TYPE;
4573
4574         if (fconf & F_MACMATCH)
4575                 mode |= T4_FILTER_MAC_IDX;
4576
4577         if (fconf & F_ETHERTYPE)
4578                 mode |= T4_FILTER_ETH_TYPE;
4579
4580         if (fconf & F_PROTOCOL)
4581                 mode |= T4_FILTER_IP_PROTO;
4582
4583         if (fconf & F_TOS)
4584                 mode |= T4_FILTER_IP_TOS;
4585
4586         if (fconf & F_VLAN)
4587                 mode |= T4_FILTER_VLAN;
4588
4589         if (fconf & F_VNIC_ID)
4590                 mode |= T4_FILTER_VNIC;
4591
4592         if (fconf & F_PORT)
4593                 mode |= T4_FILTER_PORT;
4594
4595         if (fconf & F_FCOE)
4596                 mode |= T4_FILTER_FCoE;
4597
4598         return (mode);
4599 }
4600
4601 static uint32_t
4602 mode_to_fconf(uint32_t mode)
4603 {
4604         uint32_t fconf = 0;
4605
4606         if (mode & T4_FILTER_IP_FRAGMENT)
4607                 fconf |= F_FRAGMENTATION;
4608
4609         if (mode & T4_FILTER_MPS_HIT_TYPE)
4610                 fconf |= F_MPSHITTYPE;
4611
4612         if (mode & T4_FILTER_MAC_IDX)
4613                 fconf |= F_MACMATCH;
4614
4615         if (mode & T4_FILTER_ETH_TYPE)
4616                 fconf |= F_ETHERTYPE;
4617
4618         if (mode & T4_FILTER_IP_PROTO)
4619                 fconf |= F_PROTOCOL;
4620
4621         if (mode & T4_FILTER_IP_TOS)
4622                 fconf |= F_TOS;
4623
4624         if (mode & T4_FILTER_VLAN)
4625                 fconf |= F_VLAN;
4626
4627         if (mode & T4_FILTER_VNIC)
4628                 fconf |= F_VNIC_ID;
4629
4630         if (mode & T4_FILTER_PORT)
4631                 fconf |= F_PORT;
4632
4633         if (mode & T4_FILTER_FCoE)
4634                 fconf |= F_FCOE;
4635
4636         return (fconf);
4637 }
4638
4639 static uint32_t
4640 fspec_to_fconf(struct t4_filter_specification *fs)
4641 {
4642         uint32_t fconf = 0;
4643
4644         if (fs->val.frag || fs->mask.frag)
4645                 fconf |= F_FRAGMENTATION;
4646
4647         if (fs->val.matchtype || fs->mask.matchtype)
4648                 fconf |= F_MPSHITTYPE;
4649
4650         if (fs->val.macidx || fs->mask.macidx)
4651                 fconf |= F_MACMATCH;
4652
4653         if (fs->val.ethtype || fs->mask.ethtype)
4654                 fconf |= F_ETHERTYPE;
4655
4656         if (fs->val.proto || fs->mask.proto)
4657                 fconf |= F_PROTOCOL;
4658
4659         if (fs->val.tos || fs->mask.tos)
4660                 fconf |= F_TOS;
4661
4662         if (fs->val.vlan_vld || fs->mask.vlan_vld)
4663                 fconf |= F_VLAN;
4664
4665         if (fs->val.vnic_vld || fs->mask.vnic_vld)
4666                 fconf |= F_VNIC_ID;
4667
4668         if (fs->val.iport || fs->mask.iport)
4669                 fconf |= F_PORT;
4670
4671         if (fs->val.fcoe || fs->mask.fcoe)
4672                 fconf |= F_FCOE;
4673
4674         return (fconf);
4675 }
4676
4677 static int
4678 get_filter_mode(struct adapter *sc, uint32_t *mode)
4679 {
4680         uint32_t fconf;
4681
4682         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
4683             A_TP_VLAN_PRI_MAP);
4684
4685         if (sc->filter_mode != fconf) {
4686                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
4687                     device_get_nameunit(sc->dev), sc->filter_mode, fconf);
4688                 sc->filter_mode = fconf;
4689         }
4690
4691         *mode = fconf_to_mode(sc->filter_mode);
4692
4693         return (0);
4694 }
4695
4696 static int
4697 set_filter_mode(struct adapter *sc, uint32_t mode)
4698 {
4699         uint32_t fconf;
4700         int rc;
4701
4702         fconf = mode_to_fconf(mode);
4703
4704         ADAPTER_LOCK(sc);
4705         if (IS_BUSY(sc)) {
4706                 rc = EAGAIN;
4707                 goto done;
4708         }
4709
4710         if (sc->tids.ftids_in_use > 0) {
4711                 rc = EBUSY;
4712                 goto done;
4713         }
4714
4715 #ifdef TCP_OFFLOAD
4716         if (sc->offload_map) {
4717                 rc = EBUSY;
4718                 goto done;
4719         }
4720 #endif
4721
4722 #ifdef notyet
4723         rc = -t4_set_filter_mode(sc, fconf);
4724         if (rc == 0)
4725                 sc->filter_mode = fconf;
4726 #else
4727         rc = ENOTSUP;
4728 #endif
4729
4730 done:
4731         ADAPTER_UNLOCK(sc);
4732         return (rc);
4733 }
4734
4735 static inline uint64_t
4736 get_filter_hits(struct adapter *sc, uint32_t fid)
4737 {
4738         uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4739         uint64_t hits;
4740
4741         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
4742             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
4743         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
4744         hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
4745
4746         return (be64toh(hits));
4747 }
4748
4749 static int
4750 get_filter(struct adapter *sc, struct t4_filter *t)
4751 {
4752         int i, nfilters = sc->tids.nftids;
4753         struct filter_entry *f;
4754
4755         ADAPTER_LOCK_ASSERT_OWNED(sc);
4756
4757         if (IS_BUSY(sc))
4758                 return (EAGAIN);
4759
4760         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
4761             t->idx >= nfilters) {
4762                 t->idx = 0xffffffff;
4763                 return (0);
4764         }
4765
4766         f = &sc->tids.ftid_tab[t->idx];
4767         for (i = t->idx; i < nfilters; i++, f++) {
4768                 if (f->valid) {
4769                         t->idx = i;
4770                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
4771                         t->smtidx = f->smtidx;
4772                         if (f->fs.hitcnts)
4773                                 t->hits = get_filter_hits(sc, t->idx);
4774                         else
4775                                 t->hits = UINT64_MAX;
4776                         t->fs = f->fs;
4777
4778                         return (0);
4779                 }
4780         }
4781
4782         t->idx = 0xffffffff;
4783         return (0);
4784 }
4785
4786 static int
4787 set_filter(struct adapter *sc, struct t4_filter *t)
4788 {
4789         unsigned int nfilters, nports;
4790         struct filter_entry *f;
4791         int i;
4792
4793         ADAPTER_LOCK_ASSERT_OWNED(sc);
4794
4795         nfilters = sc->tids.nftids;
4796         nports = sc->params.nports;
4797
4798         if (nfilters == 0)
4799                 return (ENOTSUP);
4800
4801         if (!(sc->flags & FULL_INIT_DONE))
4802                 return (EAGAIN);
4803
4804         if (t->idx >= nfilters)
4805                 return (EINVAL);
4806
4807         /* Validate against the global filter mode */
4808         if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode)
4809                 return (E2BIG);
4810
4811         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports)
4812                 return (EINVAL);
4813
4814         if (t->fs.val.iport >= nports)
4815                 return (EINVAL);
4816
4817         /* Can't specify an iq if not steering to it */
4818         if (!t->fs.dirsteer && t->fs.iq)
4819                 return (EINVAL);
4820
4821         /* IPv6 filter idx must be 4 aligned */
4822         if (t->fs.type == 1 &&
4823             ((t->idx & 0x3) || t->idx + 4 >= nfilters))
4824                 return (EINVAL);
4825
4826         if (sc->tids.ftid_tab == NULL) {
4827                 KASSERT(sc->tids.ftids_in_use == 0,
4828                     ("%s: no memory allocated but filters_in_use > 0",
4829                     __func__));
4830
4831                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
4832                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
4833                 if (sc->tids.ftid_tab == NULL)
4834                         return (ENOMEM);
4835         }
4836
4837         for (i = 0; i < 4; i++) {
4838                 f = &sc->tids.ftid_tab[t->idx + i];
4839
4840                 if (f->pending || f->valid)
4841                         return (EBUSY);
4842                 if (f->locked)
4843                         return (EPERM);
4844
4845                 if (t->fs.type == 0)
4846                         break;
4847         }
4848
4849         f = &sc->tids.ftid_tab[t->idx];
4850         f->fs = t->fs;
4851
4852         return set_filter_wr(sc, t->idx);
4853 }
4854
4855 static int
4856 del_filter(struct adapter *sc, struct t4_filter *t)
4857 {
4858         unsigned int nfilters;
4859         struct filter_entry *f;
4860
4861         ADAPTER_LOCK_ASSERT_OWNED(sc);
4862
4863         if (IS_BUSY(sc))
4864                 return (EAGAIN);
4865
4866         nfilters = sc->tids.nftids;
4867
4868         if (nfilters == 0)
4869                 return (ENOTSUP);
4870
4871         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
4872             t->idx >= nfilters)
4873                 return (EINVAL);
4874
4875         if (!(sc->flags & FULL_INIT_DONE))
4876                 return (EAGAIN);
4877
4878         f = &sc->tids.ftid_tab[t->idx];
4879
4880         if (f->pending)
4881                 return (EBUSY);
4882         if (f->locked)
4883                 return (EPERM);
4884
4885         if (f->valid) {
4886                 t->fs = f->fs;  /* extra info for the caller */
4887                 return del_filter_wr(sc, t->idx);
4888         }
4889
4890         return (0);
4891 }
4892
4893 static void
4894 clear_filter(struct filter_entry *f)
4895 {
4896         if (f->l2t)
4897                 t4_l2t_release(f->l2t);
4898
4899         bzero(f, sizeof (*f));
4900 }
4901
4902 static int
4903 set_filter_wr(struct adapter *sc, int fidx)
4904 {
4905         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4906         struct wrqe *wr;
4907         struct fw_filter_wr *fwr;
4908         unsigned int ftid;
4909
4910         ADAPTER_LOCK_ASSERT_OWNED(sc);
4911
4912         if (f->fs.newdmac || f->fs.newvlan) {
4913                 /* This filter needs an L2T entry; allocate one. */
4914                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
4915                 if (f->l2t == NULL)
4916                         return (EAGAIN);
4917                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
4918                     f->fs.dmac)) {
4919                         t4_l2t_release(f->l2t);
4920                         f->l2t = NULL;
4921                         return (ENOMEM);
4922                 }
4923         }
4924
4925         ftid = sc->tids.ftid_base + fidx;
4926
4927         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
4928         if (wr == NULL)
4929                 return (ENOMEM);
4930
4931         fwr = wrtod(wr);
4932         bzero(fwr, sizeof (*fwr));
4933
4934         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
4935         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
4936         fwr->tid_to_iq =
4937             htobe32(V_FW_FILTER_WR_TID(ftid) |
4938                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
4939                 V_FW_FILTER_WR_NOREPLY(0) |
4940                 V_FW_FILTER_WR_IQ(f->fs.iq));
4941         fwr->del_filter_to_l2tix =
4942             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
4943                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
4944                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
4945                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
4946                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
4947                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
4948                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
4949                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
4950                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
4951                     f->fs.newvlan == VLAN_REWRITE) |
4952                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
4953                     f->fs.newvlan == VLAN_REWRITE) |
4954                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
4955                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
4956                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
4957                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
4958         fwr->ethtype = htobe16(f->fs.val.ethtype);
4959         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
4960         fwr->frag_to_ovlan_vldm =
4961             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
4962                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
4963                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
4964                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
4965                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
4966                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
4967         fwr->smac_sel = 0;
4968         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
4969             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
4970         fwr->maci_to_matchtypem =
4971             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
4972                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
4973                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
4974                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
4975                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
4976                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
4977                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
4978                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
4979         fwr->ptcl = f->fs.val.proto;
4980         fwr->ptclm = f->fs.mask.proto;
4981         fwr->ttyp = f->fs.val.tos;
4982         fwr->ttypm = f->fs.mask.tos;
4983         fwr->ivlan = htobe16(f->fs.val.vlan);
4984         fwr->ivlanm = htobe16(f->fs.mask.vlan);
4985         fwr->ovlan = htobe16(f->fs.val.vnic);
4986         fwr->ovlanm = htobe16(f->fs.mask.vnic);
4987         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
4988         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
4989         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
4990         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
4991         fwr->lp = htobe16(f->fs.val.dport);
4992         fwr->lpm = htobe16(f->fs.mask.dport);
4993         fwr->fp = htobe16(f->fs.val.sport);
4994         fwr->fpm = htobe16(f->fs.mask.sport);
4995         if (f->fs.newsmac)
4996                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
4997
4998         f->pending = 1;
4999         sc->tids.ftids_in_use++;
5000
5001         t4_wrq_tx(sc, wr);
5002         return (0);
5003 }
5004
5005 static int
5006 del_filter_wr(struct adapter *sc, int fidx)
5007 {
5008         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
5009         struct wrqe *wr;
5010         struct fw_filter_wr *fwr;
5011         unsigned int ftid;
5012
5013         ADAPTER_LOCK_ASSERT_OWNED(sc);
5014
5015         ftid = sc->tids.ftid_base + fidx;
5016
5017         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5018         if (wr == NULL)
5019                 return (ENOMEM);
5020         fwr = wrtod(wr);
5021         bzero(fwr, sizeof (*fwr));
5022
5023         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
5024
5025         f->pending = 1;
5026         t4_wrq_tx(sc, wr);
5027         return (0);
5028 }
5029
5030 int
5031 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5032 {
5033         struct adapter *sc = iq->adapter;
5034         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
5035         unsigned int idx = GET_TID(rpl);
5036
5037         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5038             rss->opcode));
5039
5040         if (idx >= sc->tids.ftid_base &&
5041             (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
5042                 unsigned int rc = G_COOKIE(rpl->cookie);
5043                 struct filter_entry *f = &sc->tids.ftid_tab[idx];
5044
5045                 ADAPTER_LOCK(sc);
5046                 if (rc == FW_FILTER_WR_FLT_ADDED) {
5047                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
5048                         f->pending = 0;  /* asynchronous setup completed */
5049                         f->valid = 1;
5050                 } else {
5051                         if (rc != FW_FILTER_WR_FLT_DELETED) {
5052                                 /* Add or delete failed, display an error */
5053                                 log(LOG_ERR,
5054                                     "filter %u setup failed with error %u\n",
5055                                     idx, rc);
5056                         }
5057
5058                         clear_filter(f);
5059                         sc->tids.ftids_in_use--;
5060                 }
5061                 ADAPTER_UNLOCK(sc);
5062         }
5063
5064         return (0);
5065 }
5066
5067 static int
5068 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
5069 {
5070         int rc = EINVAL;
5071
5072         if (cntxt->cid > M_CTXTQID)
5073                 return (rc);
5074
5075         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
5076             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
5077                 return (rc);
5078
5079         if (sc->flags & FW_OK) {
5080                 ADAPTER_LOCK(sc);       /* Avoid parallel t4_wr_mbox */
5081                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
5082                     &cntxt->data[0]);
5083                 ADAPTER_UNLOCK(sc);
5084         }
5085
5086         if (rc != 0) {
5087                 /* Read via firmware failed or wasn't even attempted */
5088
5089                 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
5090                     &cntxt->data[0]);
5091         }
5092
5093         return (rc);
5094 }
5095
5096 static int
5097 read_card_mem(struct adapter *sc, struct t4_mem_range *mr)
5098 {
5099         uint32_t base, size, lo, hi, win, off, remaining, i, n;
5100         uint32_t *buf, *b;
5101         int rc;
5102
5103         /* reads are in multiples of 32 bits */
5104         if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
5105                 return (EINVAL);
5106
5107         /*
5108          * We don't want to deal with potential holes so we mandate that the
5109          * requested region must lie entirely within one of the 3 memories.
5110          */
5111         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5112         if (lo & F_EDRAM0_ENABLE) {
5113                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5114                 base = G_EDRAM0_BASE(hi) << 20;
5115                 size = G_EDRAM0_SIZE(hi) << 20;
5116                 if (size > 0 &&
5117                     mr->addr >= base && mr->addr < base + size &&
5118                     mr->addr + mr->len <= base + size)
5119                         goto proceed;
5120         }
5121         if (lo & F_EDRAM1_ENABLE) {
5122                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5123                 base = G_EDRAM1_BASE(hi) << 20;
5124                 size = G_EDRAM1_SIZE(hi) << 20;
5125                 if (size > 0 &&
5126                     mr->addr >= base && mr->addr < base + size &&
5127                     mr->addr + mr->len <= base + size)
5128                         goto proceed;
5129         }
5130         if (lo & F_EXT_MEM_ENABLE) {
5131                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5132                 base = G_EXT_MEM_BASE(hi) << 20;
5133                 size = G_EXT_MEM_SIZE(hi) << 20;
5134                 if (size > 0 &&
5135                     mr->addr >= base && mr->addr < base + size &&
5136                     mr->addr + mr->len <= base + size)
5137                         goto proceed;
5138         }
5139         return (ENXIO);
5140
5141 proceed:
5142         buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5143
5144         /*
5145          * Position the PCIe window (we use memwin2) to the 16B aligned area
5146          * just at/before the requested region.
5147          */
5148         win = mr->addr & ~0xf;
5149         off = mr->addr - win;  /* offset of the requested region in the win */
5150         remaining = mr->len;
5151
5152         while (remaining) {
5153                 t4_write_reg(sc,
5154                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5155                 t4_read_reg(sc,
5156                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5157
5158                 /* number of bytes that we'll copy in the inner loop */
5159                 n = min(remaining, MEMWIN2_APERTURE - off);
5160
5161                 for (i = 0; i < n; i += 4, remaining -= 4)
5162                         *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5163
5164                 win += MEMWIN2_APERTURE;
5165                 off = 0;
5166         }
5167
5168         rc = copyout(buf, mr->data, mr->len);
5169         free(buf, M_CXGBE);
5170
5171         return (rc);
5172 }
5173
5174 static int
5175 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
5176 {
5177         int rc;
5178
5179         ADAPTER_LOCK_ASSERT_OWNED(sc);  /* for mbox */
5180
5181         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
5182                 return (EINVAL);
5183
5184         if (i2cd->len > 1) {
5185                 /* XXX: need fw support for longer reads in one go */
5186                 return (ENOTSUP);
5187         }
5188
5189         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
5190             i2cd->offset, &i2cd->data[0]);
5191
5192         return (rc);
5193 }
5194
5195 int
5196 t4_os_find_pci_capability(struct adapter *sc, int cap)
5197 {
5198         int i;
5199
5200         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5201 }
5202
5203 int
5204 t4_os_pci_save_state(struct adapter *sc)
5205 {
5206         device_t dev;
5207         struct pci_devinfo *dinfo;
5208
5209         dev = sc->dev;
5210         dinfo = device_get_ivars(dev);
5211
5212         pci_cfg_save(dev, dinfo, 0);
5213         return (0);
5214 }
5215
5216 int
5217 t4_os_pci_restore_state(struct adapter *sc)
5218 {
5219         device_t dev;
5220         struct pci_devinfo *dinfo;
5221
5222         dev = sc->dev;
5223         dinfo = device_get_ivars(dev);
5224
5225         pci_cfg_restore(dev, dinfo);
5226         return (0);
5227 }
5228
5229 void
5230 t4_os_portmod_changed(const struct adapter *sc, int idx)
5231 {
5232         struct port_info *pi = sc->port[idx];
5233         static const char *mod_str[] = {
5234                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5235         };
5236
5237         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5238                 if_printf(pi->ifp, "transceiver unplugged.\n");
5239         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5240                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
5241         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5242                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5243         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
5244                 if_printf(pi->ifp, "%s transceiver inserted.\n",
5245                     mod_str[pi->mod_type]);
5246         } else {
5247                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5248                     pi->mod_type);
5249         }
5250 }
5251
5252 void
5253 t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5254 {
5255         struct port_info *pi = sc->port[idx];
5256         struct ifnet *ifp = pi->ifp;
5257
5258         if (link_stat) {
5259                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5260                 if_link_state_change(ifp, LINK_STATE_UP);
5261         } else
5262                 if_link_state_change(ifp, LINK_STATE_DOWN);
5263 }
5264
5265 void
5266 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
5267 {
5268         struct adapter *sc;
5269
5270         mtx_lock(&t4_list_lock);
5271         SLIST_FOREACH(sc, &t4_list, link) {
5272                 /*
5273                  * func should not make any assumptions about what state sc is
5274                  * in - the only guarantee is that sc->sc_lock is a valid lock.
5275                  */
5276                 func(sc, arg);
5277         }
5278         mtx_unlock(&t4_list_lock);
5279 }
5280
5281 static int
5282 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
5283 {
5284        return (0);
5285 }
5286
5287 static int
5288 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
5289 {
5290        return (0);
5291 }
5292
5293 static int
5294 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5295     struct thread *td)
5296 {
5297         int rc;
5298         struct adapter *sc = dev->si_drv1;
5299
5300         rc = priv_check(td, PRIV_DRIVER);
5301         if (rc != 0)
5302                 return (rc);
5303
5304         switch (cmd) {
5305         case CHELSIO_T4_GETREG: {
5306                 struct t4_reg *edata = (struct t4_reg *)data;
5307
5308                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5309                         return (EFAULT);
5310
5311                 if (edata->size == 4)
5312                         edata->val = t4_read_reg(sc, edata->addr);
5313                 else if (edata->size == 8)
5314                         edata->val = t4_read_reg64(sc, edata->addr);
5315                 else
5316                         return (EINVAL);
5317
5318                 break;
5319         }
5320         case CHELSIO_T4_SETREG: {
5321                 struct t4_reg *edata = (struct t4_reg *)data;
5322
5323                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5324                         return (EFAULT);
5325
5326                 if (edata->size == 4) {
5327                         if (edata->val & 0xffffffff00000000)
5328                                 return (EINVAL);
5329                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5330                 } else if (edata->size == 8)
5331                         t4_write_reg64(sc, edata->addr, edata->val);
5332                 else
5333                         return (EINVAL);
5334                 break;
5335         }
5336         case CHELSIO_T4_REGDUMP: {
5337                 struct t4_regdump *regs = (struct t4_regdump *)data;
5338                 int reglen = T4_REGDUMP_SIZE;
5339                 uint8_t *buf;
5340
5341                 if (regs->len < reglen) {
5342                         regs->len = reglen; /* hint to the caller */
5343                         return (ENOBUFS);
5344                 }
5345
5346                 regs->len = reglen;
5347                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5348                 t4_get_regs(sc, regs, buf);
5349                 rc = copyout(buf, regs->data, reglen);
5350                 free(buf, M_CXGBE);
5351                 break;
5352         }
5353         case CHELSIO_T4_GET_FILTER_MODE:
5354                 rc = get_filter_mode(sc, (uint32_t *)data);
5355                 break;
5356         case CHELSIO_T4_SET_FILTER_MODE:
5357                 rc = set_filter_mode(sc, *(uint32_t *)data);
5358                 break;
5359         case CHELSIO_T4_GET_FILTER:
5360                 ADAPTER_LOCK(sc);
5361                 rc = get_filter(sc, (struct t4_filter *)data);
5362                 ADAPTER_UNLOCK(sc);
5363                 break;
5364         case CHELSIO_T4_SET_FILTER:
5365                 ADAPTER_LOCK(sc);
5366                 rc = set_filter(sc, (struct t4_filter *)data);
5367                 ADAPTER_UNLOCK(sc);
5368                 break;
5369         case CHELSIO_T4_DEL_FILTER:
5370                 ADAPTER_LOCK(sc);
5371                 rc = del_filter(sc, (struct t4_filter *)data);
5372                 ADAPTER_UNLOCK(sc);
5373                 break;
5374         case CHELSIO_T4_GET_SGE_CONTEXT:
5375                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
5376                 break;
5377         case CHELSIO_T4_LOAD_FW: {
5378                 struct t4_data *fw = (struct t4_data *)data;
5379                 uint8_t *fw_data;
5380
5381                 if (sc->flags & FULL_INIT_DONE)
5382                         return (EBUSY);
5383
5384                 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT);
5385                 if (fw_data == NULL)
5386                         return (ENOMEM);
5387
5388                 rc = copyin(fw->data, fw_data, fw->len);
5389                 if (rc == 0)
5390                         rc = -t4_load_fw(sc, fw_data, fw->len);
5391
5392                 free(fw_data, M_CXGBE);
5393                 break;
5394         }
5395         case CHELSIO_T4_GET_MEM:
5396                 rc = read_card_mem(sc, (struct t4_mem_range *)data);
5397                 break;
5398         case CHELSIO_T4_GET_I2C:
5399                 ADAPTER_LOCK(sc);
5400                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
5401                 ADAPTER_UNLOCK(sc);
5402                 break;
5403         default:
5404                 rc = EINVAL;
5405         }
5406
5407         return (rc);
5408 }
5409
5410 #ifdef TCP_OFFLOAD
5411 static int
5412 toe_capability(struct port_info *pi, int enable)
5413 {
5414         int rc;
5415         struct adapter *sc = pi->adapter;
5416
5417         ADAPTER_LOCK_ASSERT_OWNED(sc);
5418
5419         if (!is_offload(sc))
5420                 return (ENODEV);
5421
5422         if (enable) {
5423                 if (!(sc->flags & FULL_INIT_DONE)) {
5424                         log(LOG_WARNING,
5425                             "You must enable a cxgbe interface first\n");
5426                         return (EAGAIN);
5427                 }
5428
5429                 if (isset(&sc->offload_map, pi->port_id))
5430                         return (0);
5431
5432                 if (!(sc->flags & TOM_INIT_DONE)) {
5433                         rc = t4_activate_uld(sc, ULD_TOM);
5434                         if (rc == EAGAIN) {
5435                                 log(LOG_WARNING,
5436                                     "You must kldload t4_tom.ko before trying "
5437                                     "to enable TOE on a cxgbe interface.\n");
5438                         }
5439                         if (rc != 0)
5440                                 return (rc);
5441                         KASSERT(sc->tom_softc != NULL,
5442                             ("%s: TOM activated but softc NULL", __func__));
5443                         KASSERT(sc->flags & TOM_INIT_DONE,
5444                             ("%s: TOM activated but flag not set", __func__));
5445                 }
5446
5447                 setbit(&sc->offload_map, pi->port_id);
5448         } else {
5449                 if (!isset(&sc->offload_map, pi->port_id))
5450                         return (0);
5451
5452                 KASSERT(sc->flags & TOM_INIT_DONE,
5453                     ("%s: TOM never initialized?", __func__));
5454                 clrbit(&sc->offload_map, pi->port_id);
5455         }
5456
5457         return (0);
5458 }
5459
5460 /*
5461  * Add an upper layer driver to the global list.
5462  */
5463 int
5464 t4_register_uld(struct uld_info *ui)
5465 {
5466         int rc = 0;
5467         struct uld_info *u;
5468
5469         mtx_lock(&t4_uld_list_lock);
5470         SLIST_FOREACH(u, &t4_uld_list, link) {
5471             if (u->uld_id == ui->uld_id) {
5472                     rc = EEXIST;
5473                     goto done;
5474             }
5475         }
5476
5477         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5478         ui->refcount = 0;
5479 done:
5480         mtx_unlock(&t4_uld_list_lock);
5481         return (rc);
5482 }
5483
5484 int
5485 t4_unregister_uld(struct uld_info *ui)
5486 {
5487         int rc = EINVAL;
5488         struct uld_info *u;
5489
5490         mtx_lock(&t4_uld_list_lock);
5491
5492         SLIST_FOREACH(u, &t4_uld_list, link) {
5493             if (u == ui) {
5494                     if (ui->refcount > 0) {
5495                             rc = EBUSY;
5496                             goto done;
5497                     }
5498
5499                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5500                     rc = 0;
5501                     goto done;
5502             }
5503         }
5504 done:
5505         mtx_unlock(&t4_uld_list_lock);
5506         return (rc);
5507 }
5508
5509 int
5510 t4_activate_uld(struct adapter *sc, int id)
5511 {
5512         int rc = EAGAIN;
5513         struct uld_info *ui;
5514
5515         mtx_lock(&t4_uld_list_lock);
5516
5517         SLIST_FOREACH(ui, &t4_uld_list, link) {
5518                 if (ui->uld_id == id) {
5519                         rc = ui->activate(sc);
5520                         if (rc == 0)
5521                                 ui->refcount++;
5522                         goto done;
5523                 }
5524         }
5525 done:
5526         mtx_unlock(&t4_uld_list_lock);
5527
5528         return (rc);
5529 }
5530
5531 int
5532 t4_deactivate_uld(struct adapter *sc, int id)
5533 {
5534         int rc = EINVAL;
5535         struct uld_info *ui;
5536
5537         mtx_lock(&t4_uld_list_lock);
5538
5539         SLIST_FOREACH(ui, &t4_uld_list, link) {
5540                 if (ui->uld_id == id) {
5541                         rc = ui->deactivate(sc);
5542                         if (rc == 0)
5543                                 ui->refcount--;
5544                         goto done;
5545                 }
5546         }
5547 done:
5548         mtx_unlock(&t4_uld_list_lock);
5549
5550         return (rc);
5551 }
5552 #endif
5553
5554 /*
5555  * Come up with reasonable defaults for some of the tunables, provided they're
5556  * not set by the user (in which case we'll use the values as is).
5557  */
5558 static void
5559 tweak_tunables(void)
5560 {
5561         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
5562
5563         if (t4_ntxq10g < 1)
5564                 t4_ntxq10g = min(nc, NTXQ_10G);
5565
5566         if (t4_ntxq1g < 1)
5567                 t4_ntxq1g = min(nc, NTXQ_1G);
5568
5569         if (t4_nrxq10g < 1)
5570                 t4_nrxq10g = min(nc, NRXQ_10G);
5571
5572         if (t4_nrxq1g < 1)
5573                 t4_nrxq1g = min(nc, NRXQ_1G);
5574
5575 #ifdef TCP_OFFLOAD
5576         if (t4_nofldtxq10g < 1)
5577                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
5578
5579         if (t4_nofldtxq1g < 1)
5580                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
5581
5582         if (t4_nofldrxq10g < 1)
5583                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
5584
5585         if (t4_nofldrxq1g < 1)
5586                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
5587
5588         if (t4_toecaps_allowed == -1)
5589                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
5590 #else
5591         if (t4_toecaps_allowed == -1)
5592                 t4_toecaps_allowed = 0;
5593 #endif
5594
5595         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
5596                 t4_tmr_idx_10g = TMR_IDX_10G;
5597
5598         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
5599                 t4_pktc_idx_10g = PKTC_IDX_10G;
5600
5601         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
5602                 t4_tmr_idx_1g = TMR_IDX_1G;
5603
5604         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
5605                 t4_pktc_idx_1g = PKTC_IDX_1G;
5606
5607         if (t4_qsize_txq < 128)
5608                 t4_qsize_txq = 128;
5609
5610         if (t4_qsize_rxq < 128)
5611                 t4_qsize_rxq = 128;
5612         while (t4_qsize_rxq & 7)
5613                 t4_qsize_rxq++;
5614
5615         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
5616 }
5617
5618 static int
5619 t4_mod_event(module_t mod, int cmd, void *arg)
5620 {
5621         int rc = 0;
5622
5623         switch (cmd) {
5624         case MOD_LOAD:
5625                 t4_sge_modload();
5626                 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
5627                 SLIST_INIT(&t4_list);
5628 #ifdef TCP_OFFLOAD
5629                 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
5630                 SLIST_INIT(&t4_uld_list);
5631 #endif
5632                 tweak_tunables();
5633                 break;
5634
5635         case MOD_UNLOAD:
5636 #ifdef TCP_OFFLOAD
5637                 mtx_lock(&t4_uld_list_lock);
5638                 if (!SLIST_EMPTY(&t4_uld_list)) {
5639                         rc = EBUSY;
5640                         mtx_unlock(&t4_uld_list_lock);
5641                         break;
5642                 }
5643                 mtx_unlock(&t4_uld_list_lock);
5644                 mtx_destroy(&t4_uld_list_lock);
5645 #endif
5646                 mtx_lock(&t4_list_lock);
5647                 if (!SLIST_EMPTY(&t4_list)) {
5648                         rc = EBUSY;
5649                         mtx_unlock(&t4_list_lock);
5650                         break;
5651                 }
5652                 mtx_unlock(&t4_list_lock);
5653                 mtx_destroy(&t4_list_lock);
5654                 break;
5655         }
5656
5657         return (rc);
5658 }
5659
5660 static devclass_t t4_devclass;
5661 static devclass_t cxgbe_devclass;
5662
5663 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
5664 MODULE_VERSION(t4nex, 1);
5665
5666 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
5667 MODULE_VERSION(cxgbe, 1);