sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58
  59 #include "common/common.h"
  60 #include "common/t4_msg.h"
  61 #include "common/t4_regs.h"
  62 #include "common/t4_regs_values.h"
  63 #include "t4_ioctl.h"
  64 #include "t4_l2t.h"
  65
  66 /* T4 bus driver interface */
  67 static int t4_probe(device_t);
  68 static int t4_attach(device_t);
  69 static int t4_detach(device_t);
  70 static device_method_t t4_methods[] = {
  71         DEVMETHOD(device_probe,         t4_probe),
  72         DEVMETHOD(device_attach,        t4_attach),
  73         DEVMETHOD(device_detach,        t4_detach),
  74
  75         DEVMETHOD_END
  76 };
  77 static driver_t t4_driver = {
  78         "t4nex",
  79         t4_methods,
  80         sizeof(struct adapter)
  81 };
  82
  83
  84 /* T4 port (cxgbe) interface */
  85 static int cxgbe_probe(device_t);
  86 static int cxgbe_attach(device_t);
  87 static int cxgbe_detach(device_t);
  88 static device_method_t cxgbe_methods[] = {
  89         DEVMETHOD(device_probe,         cxgbe_probe),
  90         DEVMETHOD(device_attach,        cxgbe_attach),
  91         DEVMETHOD(device_detach,        cxgbe_detach),
  92         { 0, 0 }
  93 };
  94 static driver_t cxgbe_driver = {
  95         "cxgbe",
  96         cxgbe_methods,
  97         sizeof(struct port_info)
  98 };
  99
 100 static d_ioctl_t t4_ioctl;
 101 static d_open_t t4_open;
 102 static d_close_t t4_close;
 103
 104 static struct cdevsw t4_cdevsw = {
 105        .d_version = D_VERSION,
 106        .d_flags = 0,
 107        .d_open = t4_open,
 108        .d_close = t4_close,
 109        .d_ioctl = t4_ioctl,
 110        .d_name = "t4nex",
 111 };
 112
 113 /* ifnet + media interface */
 114 static void cxgbe_init(void *);
 115 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 116 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 117 static void cxgbe_qflush(struct ifnet *);
 118 static int cxgbe_media_change(struct ifnet *);
 119 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 120
 121 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
 122
 123 /*
 124  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 125  * then ADAPTER_LOCK, then t4_uld_list_lock.
 126  */
 127 static struct mtx t4_list_lock;
 128 static SLIST_HEAD(, adapter) t4_list;
 129 #ifdef TCP_OFFLOAD
 130 static struct mtx t4_uld_list_lock;
 131 static SLIST_HEAD(, uld_info) t4_uld_list;
 132 #endif
 133
 134 /*
 135  * Tunables.  See tweak_tunables() too.
 136  */
 137
 138 /*
 139  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 140  */
 141 #define NTXQ_10G 16
 142 static int t4_ntxq10g = -1;
 143 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 144
 145 #define NRXQ_10G 8
 146 static int t4_nrxq10g = -1;
 147 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 148
 149 #define NTXQ_1G 4
 150 static int t4_ntxq1g = -1;
 151 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 152
 153 #define NRXQ_1G 2
 154 static int t4_nrxq1g = -1;
 155 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 156
 157 #ifdef TCP_OFFLOAD
 158 #define NOFLDTXQ_10G 8
 159 static int t4_nofldtxq10g = -1;
 160 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 161
 162 #define NOFLDRXQ_10G 2
 163 static int t4_nofldrxq10g = -1;
 164 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 165
 166 #define NOFLDTXQ_1G 2
 167 static int t4_nofldtxq1g = -1;
 168 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 169
 170 #define NOFLDRXQ_1G 1
 171 static int t4_nofldrxq1g = -1;
 172 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 173 #endif
 174
 175 /*
 176  * Holdoff parameters for 10G and 1G ports.
 177  */
 178 #define TMR_IDX_10G 1
 179 static int t4_tmr_idx_10g = TMR_IDX_10G;
 180 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 181
 182 #define PKTC_IDX_10G (-1)
 183 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 184 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 185
 186 #define TMR_IDX_1G 1
 187 static int t4_tmr_idx_1g = TMR_IDX_1G;
 188 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 189
 190 #define PKTC_IDX_1G (-1)
 191 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 192 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 193
 194 /*
 195  * Size (# of entries) of each tx and rx queue.
 196  */
 197 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 198 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 199
 200 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 201 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 202
 203 /*
 204  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 205  */
 206 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 207 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 208
 209 /*
 210  * Configuration file.
 211  */
 212 static char t4_cfg_file[32] = "default";
 213 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 214
 215 /*
 216  * ASIC features that will be used.  Disable the ones you don't want so that the
 217  * chip resources aren't wasted on features that will not be used.
 218  */
 219 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 220 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 221
 222 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 223 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 224
 225 static int t4_toecaps_allowed = -1;
 226 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 227
 228 static int t4_rdmacaps_allowed = 0;
 229 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 230
 231 static int t4_iscsicaps_allowed = 0;
 232 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 233
 234 static int t4_fcoecaps_allowed = 0;
 235 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 236
 237 struct intrs_and_queues {
 238         int intr_type;          /* INTx, MSI, or MSI-X */
 239         int nirq;               /* Number of vectors */
 240         int intr_flags;
 241         int ntxq10g;            /* # of NIC txq's for each 10G port */
 242         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 243         int ntxq1g;             /* # of NIC txq's for each 1G port */
 244         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 245 #ifdef TCP_OFFLOAD
 246         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 247         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 248         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 249         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 250 #endif
 251 };
 252
 253 struct filter_entry {
 254         uint32_t valid:1;       /* filter allocated and valid */
 255         uint32_t locked:1;      /* filter is administratively locked */
 256         uint32_t pending:1;     /* filter action is pending firmware reply */
 257         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 258         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 259
 260         struct t4_filter_specification fs;
 261 };
 262
 263 enum {
 264         XGMAC_MTU       = (1 << 0),
 265         XGMAC_PROMISC   = (1 << 1),
 266         XGMAC_ALLMULTI  = (1 << 2),
 267         XGMAC_VLANEX    = (1 << 3),
 268         XGMAC_UCADDR    = (1 << 4),
 269         XGMAC_MCADDRS   = (1 << 5),
 270
 271         XGMAC_ALL       = 0xffff
 272 };
 273
 274 static int map_bars(struct adapter *);
 275 static void setup_memwin(struct adapter *);
 276 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 277     struct intrs_and_queues *);
 278 static int prep_firmware(struct adapter *);
 279 static int upload_config_file(struct adapter *, const struct firmware *,
 280     uint32_t *, uint32_t *);
 281 static int partition_resources(struct adapter *, const struct firmware *);
 282 static int get_params__pre_init(struct adapter *);
 283 static int get_params__post_init(struct adapter *);
 284 static void t4_set_desc(struct adapter *);
 285 static void build_medialist(struct port_info *);
 286 static int update_mac_settings(struct port_info *, int);
 287 static int cxgbe_init_synchronized(struct port_info *);
 288 static int cxgbe_uninit_synchronized(struct port_info *);
 289 static int setup_intr_handlers(struct adapter *);
 290 static int adapter_full_init(struct adapter *);
 291 static int adapter_full_uninit(struct adapter *);
 292 static int port_full_init(struct port_info *);
 293 static int port_full_uninit(struct port_info *);
 294 static void quiesce_eq(struct adapter *, struct sge_eq *);
 295 static void quiesce_iq(struct adapter *, struct sge_iq *);
 296 static void quiesce_fl(struct adapter *, struct sge_fl *);
 297 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 298     driver_intr_t *, void *, char *);
 299 static int t4_free_irq(struct adapter *, struct irq *);
 300 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 301     unsigned int);
 302 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 303 static void cxgbe_tick(void *);
 304 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 305 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 306     struct mbuf *);
 307 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 308 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 309 static int t4_sysctls(struct adapter *);
 310 static int cxgbe_sysctls(struct port_info *);
 311 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 312 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 313 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 314 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 315 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 316 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 317 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 318 #ifdef SBUF_DRAIN
 319 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 320 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 321 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 322 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 323 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 324 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 325 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 326 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 327 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 328 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 329 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 330 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 331 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 332 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 333 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 334 #endif
 335 static inline void txq_start(struct ifnet *, struct sge_txq *);
 336 static uint32_t fconf_to_mode(uint32_t);
 337 static uint32_t mode_to_fconf(uint32_t);
 338 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 339 static int get_filter_mode(struct adapter *, uint32_t *);
 340 static int set_filter_mode(struct adapter *, uint32_t);
 341 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 342 static int get_filter(struct adapter *, struct t4_filter *);
 343 static int set_filter(struct adapter *, struct t4_filter *);
 344 static int del_filter(struct adapter *, struct t4_filter *);
 345 static void clear_filter(struct filter_entry *);
 346 static int set_filter_wr(struct adapter *, int);
 347 static int del_filter_wr(struct adapter *, int);
 348 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 349 static int load_fw(struct adapter *, struct t4_data *);
 350 static int read_card_mem(struct adapter *, struct t4_mem_range *);
 351 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 352 #ifdef TCP_OFFLOAD
 353 static int toe_capability(struct port_info *, int);
 354 #endif
 355 static int t4_mod_event(module_t, int, void *);
 356
 357 struct t4_pciids {
 358         uint16_t device;
 359         char *desc;
 360 } t4_pciids[] = {
 361         {0xa000, "Chelsio Terminator 4 FPGA"},
 362         {0x4400, "Chelsio T440-dbg"},
 363         {0x4401, "Chelsio T420-CR"},
 364         {0x4402, "Chelsio T422-CR"},
 365         {0x4403, "Chelsio T440-CR"},
 366         {0x4404, "Chelsio T420-BCH"},
 367         {0x4405, "Chelsio T440-BCH"},
 368         {0x4406, "Chelsio T440-CH"},
 369         {0x4407, "Chelsio T420-SO"},
 370         {0x4408, "Chelsio T420-CX"},
 371         {0x4409, "Chelsio T420-BT"},
 372         {0x440a, "Chelsio T404-BT"},
 373         {0x440e, "Chelsio T440-LP-CR"},
 374 };
 375
 376 #ifdef TCP_OFFLOAD
 377 /*
 378  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 379  * exactly the same for both rxq and ofld_rxq.
 380  */
 381 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 382 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 383 #endif
 384
 385 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 386 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 387 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 388
 389 static int
 390 t4_probe(device_t dev)
 391 {
 392         int i;
 393         uint16_t v = pci_get_vendor(dev);
 394         uint16_t d = pci_get_device(dev);
 395         uint8_t f = pci_get_function(dev);
 396
 397         if (v != PCI_VENDOR_ID_CHELSIO)
 398                 return (ENXIO);
 399
 400         /* Attach only to PF0 of the FPGA */
 401         if (d == 0xa000 && f != 0)
 402                 return (ENXIO);
 403
 404         for (i = 0; i < nitems(t4_pciids); i++) {
 405                 if (d == t4_pciids[i].device) {
 406                         device_set_desc(dev, t4_pciids[i].desc);
 407                         return (BUS_PROBE_DEFAULT);
 408                 }
 409         }
 410
 411         return (ENXIO);
 412 }
 413
 414 static int
 415 t4_attach(device_t dev)
 416 {
 417         struct adapter *sc;
 418         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 419         struct intrs_and_queues iaq;
 420         struct sge *s;
 421 #ifdef TCP_OFFLOAD
 422         int ofld_rqidx, ofld_tqidx;
 423 #endif
 424
 425         sc = device_get_softc(dev);
 426         sc->dev = dev;
 427
 428         pci_enable_busmaster(dev);
 429         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 430                 uint32_t v;
 431
 432                 pci_set_max_read_req(dev, 4096);
 433                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 434                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 435                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 436         }
 437
 438         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 439             device_get_nameunit(dev));
 440         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 441         mtx_lock(&t4_list_lock);
 442         SLIST_INSERT_HEAD(&t4_list, sc, link);
 443         mtx_unlock(&t4_list_lock);
 444
 445         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 446         TAILQ_INIT(&sc->sfl);
 447         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 448
 449         rc = map_bars(sc);
 450         if (rc != 0)
 451                 goto done; /* error message displayed already */
 452
 453         /*
 454          * This is the real PF# to which we're attaching.  Works from within PCI
 455          * passthrough environments too, where pci_get_function() could return a
 456          * different PF# depending on the passthrough configuration.  We need to
 457          * use the real PF# in all our communication with the firmware.
 458          */
 459         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 460         sc->mbox = sc->pf;
 461
 462         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 463         sc->an_handler = an_not_handled;
 464         for (i = 0; i < nitems(sc->cpl_handler); i++)
 465                 sc->cpl_handler[i] = cpl_not_handled;
 466         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 467                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 468         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 469
 470         /* Prepare the adapter for operation */
 471         rc = -t4_prep_adapter(sc);
 472         if (rc != 0) {
 473                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 474                 goto done;
 475         }
 476
 477         /*
 478          * Do this really early, with the memory windows set up even before the
 479          * character device.  The userland tool's register i/o and mem read
 480          * will work even in "recovery mode".
 481          */
 482         setup_memwin(sc);
 483         sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
 484             GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
 485         sc->cdev->si_drv1 = sc;
 486
 487         /* Go no further if recovery mode has been requested. */
 488         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 489                 device_printf(dev, "recovery mode.\n");
 490                 goto done;
 491         }
 492
 493         /* Prepare the firmware for operation */
 494         rc = prep_firmware(sc);
 495         if (rc != 0)
 496                 goto done; /* error message displayed already */
 497
 498         rc = get_params__pre_init(sc);
 499         if (rc != 0)
 500                 goto done; /* error message displayed already */
 501
 502         rc = t4_sge_init(sc);
 503         if (rc != 0)
 504                 goto done; /* error message displayed already */
 505
 506         if (sc->flags & MASTER_PF) {
 507                 /* get basic stuff going */
 508                 rc = -t4_fw_initialize(sc, sc->mbox);
 509                 if (rc != 0) {
 510                         device_printf(dev, "early init failed: %d.\n", rc);
 511                         goto done;
 512                 }
 513         }
 514
 515         rc = get_params__post_init(sc);
 516         if (rc != 0)
 517                 goto done; /* error message displayed already */
 518
 519         if (sc->flags & MASTER_PF) {
 520                 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
 521
 522                 /* final tweaks to some settings */
 523
 524                 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
 525                     sc->params.b_wnd);
 526                 /* 4K, 16K, 64K, 256K DDP "page sizes" */
 527                 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) |
 528                     V_HPZ2(4) | V_HPZ3(6));
 529                 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB);
 530                 t4_set_reg_field(sc, A_TP_PARA_REG5,
 531                     V_INDICATESIZE(M_INDICATESIZE) |
 532                     F_REARMDDPOFFSET | F_RESETDDPOFFSET,
 533                     V_INDICATESIZE(indsz) |
 534                     F_REARMDDPOFFSET | F_RESETDDPOFFSET);
 535         } else {
 536                 /*
 537                  * XXX: Verify that we can live with whatever the master driver
 538                  * has done so far, and hope that it doesn't change any global
 539                  * setting from underneath us in the future.
 540                  */
 541         }
 542
 543         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
 544             A_TP_VLAN_PRI_MAP);
 545
 546         for (i = 0; i < NCHAN; i++)
 547                 sc->params.tp.tx_modq[i] = i;
 548
 549         rc = t4_create_dma_tag(sc);
 550         if (rc != 0)
 551                 goto done; /* error message displayed already */
 552
 553         /*
 554          * First pass over all the ports - allocate VIs and initialize some
 555          * basic parameters like mac address, port type, etc.  We also figure
 556          * out whether a port is 10G or 1G and use that information when
 557          * calculating how many interrupts to attempt to allocate.
 558          */
 559         n10g = n1g = 0;
 560         for_each_port(sc, i) {
 561                 struct port_info *pi;
 562
 563                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 564                 sc->port[i] = pi;
 565
 566                 /* These must be set before t4_port_init */
 567                 pi->adapter = sc;
 568                 pi->port_id = i;
 569
 570                 /* Allocate the vi and initialize parameters like mac addr */
 571                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 572                 if (rc != 0) {
 573                         device_printf(dev, "unable to initialize port %d: %d\n",
 574                             i, rc);
 575                         free(pi, M_CXGBE);
 576                         sc->port[i] = NULL;
 577                         goto done;
 578                 }
 579
 580                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 581                     device_get_nameunit(dev), i);
 582                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 583
 584                 if (is_10G_port(pi)) {
 585                         n10g++;
 586                         pi->tmr_idx = t4_tmr_idx_10g;
 587                         pi->pktc_idx = t4_pktc_idx_10g;
 588                 } else {
 589                         n1g++;
 590                         pi->tmr_idx = t4_tmr_idx_1g;
 591                         pi->pktc_idx = t4_pktc_idx_1g;
 592                 }
 593
 594                 pi->xact_addr_filt = -1;
 595
 596                 pi->qsize_rxq = t4_qsize_rxq;
 597                 pi->qsize_txq = t4_qsize_txq;
 598
 599                 pi->dev = device_add_child(dev, "cxgbe", -1);
 600                 if (pi->dev == NULL) {
 601                         device_printf(dev,
 602                             "failed to add device for port %d.\n", i);
 603                         rc = ENXIO;
 604                         goto done;
 605                 }
 606                 device_set_softc(pi->dev, pi);
 607         }
 608
 609         /*
 610          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 611          */
 612         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 613         if (rc != 0)
 614                 goto done; /* error message displayed already */
 615
 616         sc->intr_type = iaq.intr_type;
 617         sc->intr_count = iaq.nirq;
 618         sc->flags |= iaq.intr_flags;
 619
 620         s = &sc->sge;
 621         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 622         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 623         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 624         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 625         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 626
 627 #ifdef TCP_OFFLOAD
 628         if (is_offload(sc)) {
 629
 630                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 631                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 632                 s->neq += s->nofldtxq + s->nofldrxq;
 633                 s->niq += s->nofldrxq;
 634
 635                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 636                     M_CXGBE, M_ZERO | M_WAITOK);
 637                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 638                     M_CXGBE, M_ZERO | M_WAITOK);
 639         }
 640 #endif
 641
 642         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 643             M_ZERO | M_WAITOK);
 644         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 645             M_ZERO | M_WAITOK);
 646         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 647             M_ZERO | M_WAITOK);
 648         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 649             M_ZERO | M_WAITOK);
 650         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 651             M_ZERO | M_WAITOK);
 652
 653         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 654             M_ZERO | M_WAITOK);
 655
 656         t4_init_l2t(sc, M_WAITOK);
 657
 658         /*
 659          * Second pass over the ports.  This time we know the number of rx and
 660          * tx queues that each port should get.
 661          */
 662         rqidx = tqidx = 0;
 663 #ifdef TCP_OFFLOAD
 664         ofld_rqidx = ofld_tqidx = 0;
 665 #endif
 666         for_each_port(sc, i) {
 667                 struct port_info *pi = sc->port[i];
 668
 669                 if (pi == NULL)
 670                         continue;
 671
 672                 pi->first_rxq = rqidx;
 673                 pi->first_txq = tqidx;
 674                 if (is_10G_port(pi)) {
 675                         pi->nrxq = iaq.nrxq10g;
 676                         pi->ntxq = iaq.ntxq10g;
 677                 } else {
 678                         pi->nrxq = iaq.nrxq1g;
 679                         pi->ntxq = iaq.ntxq1g;
 680                 }
 681
 682                 rqidx += pi->nrxq;
 683                 tqidx += pi->ntxq;
 684
 685 #ifdef TCP_OFFLOAD
 686                 if (is_offload(sc)) {
 687                         pi->first_ofld_rxq = ofld_rqidx;
 688                         pi->first_ofld_txq = ofld_tqidx;
 689                         if (is_10G_port(pi)) {
 690                                 pi->nofldrxq = iaq.nofldrxq10g;
 691                                 pi->nofldtxq = iaq.nofldtxq10g;
 692                         } else {
 693                                 pi->nofldrxq = iaq.nofldrxq1g;
 694                                 pi->nofldtxq = iaq.nofldtxq1g;
 695                         }
 696                         ofld_rqidx += pi->nofldrxq;
 697                         ofld_tqidx += pi->nofldtxq;
 698                 }
 699 #endif
 700         }
 701
 702         rc = setup_intr_handlers(sc);
 703         if (rc != 0) {
 704                 device_printf(dev,
 705                     "failed to setup interrupt handlers: %d\n", rc);
 706                 goto done;
 707         }
 708
 709         rc = bus_generic_attach(dev);
 710         if (rc != 0) {
 711                 device_printf(dev,
 712                     "failed to attach all child ports: %d\n", rc);
 713                 goto done;
 714         }
 715
 716         device_printf(dev,
 717             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 718             sc->params.pci.width, sc->params.nports, sc->intr_count,
 719             sc->intr_type == INTR_MSIX ? "MSI-X" :
 720             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 721             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 722
 723         t4_set_desc(sc);
 724
 725 done:
 726         if (rc != 0 && sc->cdev) {
 727                 /* cdev was created and so cxgbetool works; recover that way. */
 728                 device_printf(dev,
 729                     "error during attach, adapter is now in recovery mode.\n");
 730                 rc = 0;
 731         }
 732
 733         if (rc != 0)
 734                 t4_detach(dev);
 735         else
 736                 t4_sysctls(sc);
 737
 738         return (rc);
 739 }
 740
 741 /*
 742  * Idempotent
 743  */
 744 static int
 745 t4_detach(device_t dev)
 746 {
 747         struct adapter *sc;
 748         struct port_info *pi;
 749         int i, rc;
 750
 751         sc = device_get_softc(dev);
 752
 753         if (sc->flags & FULL_INIT_DONE)
 754                 t4_intr_disable(sc);
 755
 756         if (sc->cdev) {
 757                 destroy_dev(sc->cdev);
 758                 sc->cdev = NULL;
 759         }
 760
 761         rc = bus_generic_detach(dev);
 762         if (rc) {
 763                 device_printf(dev,
 764                     "failed to detach child devices: %d\n", rc);
 765                 return (rc);
 766         }
 767
 768         for (i = 0; i < sc->intr_count; i++)
 769                 t4_free_irq(sc, &sc->irq[i]);
 770
 771         for (i = 0; i < MAX_NPORTS; i++) {
 772                 pi = sc->port[i];
 773                 if (pi) {
 774                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 775                         if (pi->dev)
 776                                 device_delete_child(dev, pi->dev);
 777
 778                         mtx_destroy(&pi->pi_lock);
 779                         free(pi, M_CXGBE);
 780                 }
 781         }
 782
 783         if (sc->flags & FULL_INIT_DONE)
 784                 adapter_full_uninit(sc);
 785
 786         if (sc->flags & FW_OK)
 787                 t4_fw_bye(sc, sc->mbox);
 788
 789         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 790                 pci_release_msi(dev);
 791
 792         if (sc->regs_res)
 793                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 794                     sc->regs_res);
 795
 796         if (sc->msix_res)
 797                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 798                     sc->msix_res);
 799
 800         if (sc->l2t)
 801                 t4_free_l2t(sc->l2t);
 802
 803 #ifdef TCP_OFFLOAD
 804         free(sc->sge.ofld_rxq, M_CXGBE);
 805         free(sc->sge.ofld_txq, M_CXGBE);
 806 #endif
 807         free(sc->irq, M_CXGBE);
 808         free(sc->sge.rxq, M_CXGBE);
 809         free(sc->sge.txq, M_CXGBE);
 810         free(sc->sge.ctrlq, M_CXGBE);
 811         free(sc->sge.iqmap, M_CXGBE);
 812         free(sc->sge.eqmap, M_CXGBE);
 813         free(sc->tids.ftid_tab, M_CXGBE);
 814         t4_destroy_dma_tag(sc);
 815         if (mtx_initialized(&sc->sc_lock)) {
 816                 mtx_lock(&t4_list_lock);
 817                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 818                 mtx_unlock(&t4_list_lock);
 819                 mtx_destroy(&sc->sc_lock);
 820         }
 821
 822         if (mtx_initialized(&sc->tids.ftid_lock))
 823                 mtx_destroy(&sc->tids.ftid_lock);
 824         if (mtx_initialized(&sc->sfl_lock))
 825                 mtx_destroy(&sc->sfl_lock);
 826
 827         bzero(sc, sizeof(*sc));
 828
 829         return (0);
 830 }
 831
 832
 833 static int
 834 cxgbe_probe(device_t dev)
 835 {
 836         char buf[128];
 837         struct port_info *pi = device_get_softc(dev);
 838
 839         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 840         device_set_desc_copy(dev, buf);
 841
 842         return (BUS_PROBE_DEFAULT);
 843 }
 844
 845 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 846     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 847     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
 848 #define T4_CAP_ENABLE (T4_CAP)
 849
 850 static int
 851 cxgbe_attach(device_t dev)
 852 {
 853         struct port_info *pi = device_get_softc(dev);
 854         struct ifnet *ifp;
 855
 856         /* Allocate an ifnet and set it up */
 857         ifp = if_alloc(IFT_ETHER);
 858         if (ifp == NULL) {
 859                 device_printf(dev, "Cannot allocate ifnet\n");
 860                 return (ENOMEM);
 861         }
 862         pi->ifp = ifp;
 863         ifp->if_softc = pi;
 864
 865         callout_init(&pi->tick, CALLOUT_MPSAFE);
 866
 867         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 868         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 869
 870         ifp->if_init = cxgbe_init;
 871         ifp->if_ioctl = cxgbe_ioctl;
 872         ifp->if_transmit = cxgbe_transmit;
 873         ifp->if_qflush = cxgbe_qflush;
 874
 875         ifp->if_capabilities = T4_CAP;
 876 #ifdef TCP_OFFLOAD
 877         if (is_offload(pi->adapter))
 878                 ifp->if_capabilities |= IFCAP_TOE;
 879 #endif
 880         ifp->if_capenable = T4_CAP_ENABLE;
 881         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
 882             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
 883
 884         /* Initialize ifmedia for this port */
 885         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 886             cxgbe_media_status);
 887         build_medialist(pi);
 888
 889         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 890             EVENTHANDLER_PRI_ANY);
 891
 892         ether_ifattach(ifp, pi->hw_addr);
 893
 894 #ifdef TCP_OFFLOAD
 895         if (is_offload(pi->adapter)) {
 896                 device_printf(dev,
 897                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
 898                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
 899         } else
 900 #endif
 901                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
 902
 903         cxgbe_sysctls(pi);
 904
 905         return (0);
 906 }
 907
 908 static int
 909 cxgbe_detach(device_t dev)
 910 {
 911         struct port_info *pi = device_get_softc(dev);
 912         struct adapter *sc = pi->adapter;
 913         struct ifnet *ifp = pi->ifp;
 914
 915         /* Tell if_ioctl and if_init that the port is going away */
 916         ADAPTER_LOCK(sc);
 917         SET_DOOMED(pi);
 918         wakeup(&sc->flags);
 919         while (IS_BUSY(sc))
 920                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 921         SET_BUSY(sc);
 922 #ifdef INVARIANTS
 923         sc->last_op = "t4detach";
 924         sc->last_op_thr = curthread;
 925 #endif
 926         ADAPTER_UNLOCK(sc);
 927
 928         if (pi->vlan_c)
 929                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
 930
 931         PORT_LOCK(pi);
 932         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 933         callout_stop(&pi->tick);
 934         PORT_UNLOCK(pi);
 935         callout_drain(&pi->tick);
 936
 937         /* Let detach proceed even if these fail. */
 938         cxgbe_uninit_synchronized(pi);
 939         port_full_uninit(pi);
 940
 941         ifmedia_removeall(&pi->media);
 942         ether_ifdetach(pi->ifp);
 943         if_free(pi->ifp);
 944
 945         ADAPTER_LOCK(sc);
 946         CLR_BUSY(sc);
 947         wakeup(&sc->flags);
 948         ADAPTER_UNLOCK(sc);
 949
 950         return (0);
 951 }
 952
 953 static void
 954 cxgbe_init(void *arg)
 955 {
 956         struct port_info *pi = arg;
 957         struct adapter *sc = pi->adapter;
 958
 959         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
 960                 return;
 961         cxgbe_init_synchronized(pi);
 962         end_synchronized_op(sc, 0);
 963 }
 964
 965 static int
 966 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 967 {
 968         int rc = 0, mtu, flags;
 969         struct port_info *pi = ifp->if_softc;
 970         struct adapter *sc = pi->adapter;
 971         struct ifreq *ifr = (struct ifreq *)data;
 972         uint32_t mask;
 973
 974         switch (cmd) {
 975         case SIOCSIFMTU:
 976                 mtu = ifr->ifr_mtu;
 977                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
 978                         return (EINVAL);
 979
 980                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
 981                 if (rc)
 982                         return (rc);
 983                 ifp->if_mtu = mtu;
 984                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 985                         t4_update_fl_bufsize(ifp);
 986                         rc = update_mac_settings(pi, XGMAC_MTU);
 987                 }
 988                 end_synchronized_op(sc, 0);
 989                 break;
 990
 991         case SIOCSIFFLAGS:
 992                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
 993                 if (rc)
 994                         return (rc);
 995
 996                 if (ifp->if_flags & IFF_UP) {
 997                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 998                                 flags = pi->if_flags;
 999                                 if ((ifp->if_flags ^ flags) &
1000                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1001                                         rc = update_mac_settings(pi,
1002                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1003                                 }
1004                         } else
1005                                 rc = cxgbe_init_synchronized(pi);
1006                         pi->if_flags = ifp->if_flags;
1007                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1008                         rc = cxgbe_uninit_synchronized(pi);
1009                 end_synchronized_op(sc, 0);
1010                 break;
1011
1012         case SIOCADDMULTI:
1013         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1014                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1015                 if (rc)
1016                         return (rc);
1017                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1018                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1019                 end_synchronized_op(sc, LOCK_HELD);
1020                 break;
1021
1022         case SIOCSIFCAP:
1023                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1024                 if (rc)
1025                         return (rc);
1026
1027                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028                 if (mask & IFCAP_TXCSUM) {
1029                         ifp->if_capenable ^= IFCAP_TXCSUM;
1030                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1031
1032                         if (IFCAP_TSO4 & ifp->if_capenable &&
1033                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1034                                 ifp->if_capenable &= ~IFCAP_TSO4;
1035                                 if_printf(ifp,
1036                                     "tso4 disabled due to -txcsum.\n");
1037                         }
1038                 }
1039                 if (mask & IFCAP_TXCSUM_IPV6) {
1040                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1041                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1042
1043                         if (IFCAP_TSO6 & ifp->if_capenable &&
1044                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1045                                 ifp->if_capenable &= ~IFCAP_TSO6;
1046                                 if_printf(ifp,
1047                                     "tso6 disabled due to -txcsum6.\n");
1048                         }
1049                 }
1050                 if (mask & IFCAP_RXCSUM)
1051                         ifp->if_capenable ^= IFCAP_RXCSUM;
1052                 if (mask & IFCAP_RXCSUM_IPV6)
1053                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1054
1055                 /*
1056                  * Note that we leave CSUM_TSO alone (it is always set).  The
1057                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1058                  * sending a TSO request our way, so it's sufficient to toggle
1059                  * IFCAP_TSOx only.
1060                  */
1061                 if (mask & IFCAP_TSO4) {
1062                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1063                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1064                                 if_printf(ifp, "enable txcsum first.\n");
1065                                 rc = EAGAIN;
1066                                 goto fail;
1067                         }
1068                         ifp->if_capenable ^= IFCAP_TSO4;
1069                 }
1070                 if (mask & IFCAP_TSO6) {
1071                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1072                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1073                                 if_printf(ifp, "enable txcsum6 first.\n");
1074                                 rc = EAGAIN;
1075                                 goto fail;
1076                         }
1077                         ifp->if_capenable ^= IFCAP_TSO6;
1078                 }
1079                 if (mask & IFCAP_LRO) {
1080 #if defined(INET) || defined(INET6)
1081                         int i;
1082                         struct sge_rxq *rxq;
1083
1084                         ifp->if_capenable ^= IFCAP_LRO;
1085                         for_each_rxq(pi, i, rxq) {
1086                                 if (ifp->if_capenable & IFCAP_LRO)
1087                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1088                                 else
1089                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1090                         }
1091 #endif
1092                 }
1093 #ifdef TCP_OFFLOAD
1094                 if (mask & IFCAP_TOE) {
1095                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1096
1097                         rc = toe_capability(pi, enable);
1098                         if (rc != 0)
1099                                 goto fail;
1100
1101                         ifp->if_capenable ^= mask;
1102                 }
1103 #endif
1104                 if (mask & IFCAP_VLAN_HWTAGGING) {
1105                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1106                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1107                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1108                 }
1109                 if (mask & IFCAP_VLAN_MTU) {
1110                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1111
1112                         /* Need to find out how to disable auto-mtu-inflation */
1113                 }
1114                 if (mask & IFCAP_VLAN_HWTSO)
1115                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1116                 if (mask & IFCAP_VLAN_HWCSUM)
1117                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1118
1119 #ifdef VLAN_CAPABILITIES
1120                 VLAN_CAPABILITIES(ifp);
1121 #endif
1122 fail:
1123                 end_synchronized_op(sc, 0);
1124                 break;
1125
1126         case SIOCSIFMEDIA:
1127         case SIOCGIFMEDIA:
1128                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1129                 break;
1130
1131         default:
1132                 rc = ether_ioctl(ifp, cmd, data);
1133         }
1134
1135         return (rc);
1136 }
1137
1138 static int
1139 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1140 {
1141         struct port_info *pi = ifp->if_softc;
1142         struct adapter *sc = pi->adapter;
1143         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1144         struct buf_ring *br;
1145         int rc;
1146
1147         M_ASSERTPKTHDR(m);
1148
1149         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1150                 m_freem(m);
1151                 return (ENETDOWN);
1152         }
1153
1154         if (m->m_flags & M_FLOWID)
1155                 txq += (m->m_pkthdr.flowid % pi->ntxq);
1156         br = txq->br;
1157
1158         if (TXQ_TRYLOCK(txq) == 0) {
1159                 struct sge_eq *eq = &txq->eq;
1160
1161                 /*
1162                  * It is possible that t4_eth_tx finishes up and releases the
1163                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1164                  * need to make sure that this mbuf doesn't just sit there in
1165                  * the drbr.
1166                  */
1167
1168                 rc = drbr_enqueue(ifp, br, m);
1169                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1170                     !(eq->flags & EQ_DOOMED))
1171                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1172                 return (rc);
1173         }
1174
1175         /*
1176          * txq->m is the mbuf that is held up due to a temporary shortage of
1177          * resources and it should be put on the wire first.  Then what's in
1178          * drbr and finally the mbuf that was just passed in to us.
1179          *
1180          * Return code should indicate the fate of the mbuf that was passed in
1181          * this time.
1182          */
1183
1184         TXQ_LOCK_ASSERT_OWNED(txq);
1185         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1186
1187                 /* Queued for transmission. */
1188
1189                 rc = drbr_enqueue(ifp, br, m);
1190                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1191                 (void) t4_eth_tx(ifp, txq, m);
1192                 TXQ_UNLOCK(txq);
1193                 return (rc);
1194         }
1195
1196         /* Direct transmission. */
1197         rc = t4_eth_tx(ifp, txq, m);
1198         if (rc != 0 && txq->m)
1199                 rc = 0; /* held, will be transmitted soon (hopefully) */
1200
1201         TXQ_UNLOCK(txq);
1202         return (rc);
1203 }
1204
1205 static void
1206 cxgbe_qflush(struct ifnet *ifp)
1207 {
1208         struct port_info *pi = ifp->if_softc;
1209         struct sge_txq *txq;
1210         int i;
1211         struct mbuf *m;
1212
1213         /* queues do not exist if !PORT_INIT_DONE. */
1214         if (pi->flags & PORT_INIT_DONE) {
1215                 for_each_txq(pi, i, txq) {
1216                         TXQ_LOCK(txq);
1217                         m_freem(txq->m);
1218                         txq->m = NULL;
1219                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1220                                 m_freem(m);
1221                         TXQ_UNLOCK(txq);
1222                 }
1223         }
1224         if_qflush(ifp);
1225 }
1226
1227 static int
1228 cxgbe_media_change(struct ifnet *ifp)
1229 {
1230         struct port_info *pi = ifp->if_softc;
1231
1232         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1233
1234         return (EOPNOTSUPP);
1235 }
1236
1237 static void
1238 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1239 {
1240         struct port_info *pi = ifp->if_softc;
1241         struct ifmedia_entry *cur = pi->media.ifm_cur;
1242         int speed = pi->link_cfg.speed;
1243         int data = (pi->port_type << 8) | pi->mod_type;
1244
1245         if (cur->ifm_data != data) {
1246                 build_medialist(pi);
1247                 cur = pi->media.ifm_cur;
1248         }
1249
1250         ifmr->ifm_status = IFM_AVALID;
1251         if (!pi->link_cfg.link_ok)
1252                 return;
1253
1254         ifmr->ifm_status |= IFM_ACTIVE;
1255
1256         /* active and current will differ iff current media is autoselect. */
1257         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1258                 return;
1259
1260         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1261         if (speed == SPEED_10000)
1262                 ifmr->ifm_active |= IFM_10G_T;
1263         else if (speed == SPEED_1000)
1264                 ifmr->ifm_active |= IFM_1000_T;
1265         else if (speed == SPEED_100)
1266                 ifmr->ifm_active |= IFM_100_TX;
1267         else if (speed == SPEED_10)
1268                 ifmr->ifm_active |= IFM_10_T;
1269         else
1270                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1271                             speed));
1272 }
1273
1274 void
1275 t4_fatal_err(struct adapter *sc)
1276 {
1277         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1278         t4_intr_disable(sc);
1279         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1280             device_get_nameunit(sc->dev));
1281 }
1282
1283 static int
1284 map_bars(struct adapter *sc)
1285 {
1286         sc->regs_rid = PCIR_BAR(0);
1287         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1288             &sc->regs_rid, RF_ACTIVE);
1289         if (sc->regs_res == NULL) {
1290                 device_printf(sc->dev, "cannot map registers.\n");
1291                 return (ENXIO);
1292         }
1293         sc->bt = rman_get_bustag(sc->regs_res);
1294         sc->bh = rman_get_bushandle(sc->regs_res);
1295         sc->mmio_len = rman_get_size(sc->regs_res);
1296
1297         sc->msix_rid = PCIR_BAR(4);
1298         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1299             &sc->msix_rid, RF_ACTIVE);
1300         if (sc->msix_res == NULL) {
1301                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1302                 return (ENXIO);
1303         }
1304
1305         return (0);
1306 }
1307
1308 static void
1309 setup_memwin(struct adapter *sc)
1310 {
1311         uint32_t bar0;
1312
1313         /*
1314          * Read low 32b of bar0 indirectly via the hardware backdoor mechanism.
1315          * Works from within PCI passthrough environments too, where
1316          * rman_get_start() can return a different value.  We need to program
1317          * the memory window decoders with the actual addresses that will be
1318          * coming across the PCIe link.
1319          */
1320         bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1321         bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1322
1323         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1324                      (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1325                      V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1326
1327         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1328                      (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1329                      V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1330
1331         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1332                      (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1333                      V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1334
1335         /* flush */
1336         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1337 }
1338
1339 static int
1340 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1341     struct intrs_and_queues *iaq)
1342 {
1343         int rc, itype, navail, nrxq10g, nrxq1g, n;
1344         int nofldrxq10g = 0, nofldrxq1g = 0;
1345
1346         bzero(iaq, sizeof(*iaq));
1347
1348         iaq->ntxq10g = t4_ntxq10g;
1349         iaq->ntxq1g = t4_ntxq1g;
1350         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1351         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1352 #ifdef TCP_OFFLOAD
1353         if (is_offload(sc)) {
1354                 iaq->nofldtxq10g = t4_nofldtxq10g;
1355                 iaq->nofldtxq1g = t4_nofldtxq1g;
1356                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1357                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1358         }
1359 #endif
1360
1361         for (itype = INTR_MSIX; itype; itype >>= 1) {
1362
1363                 if ((itype & t4_intr_types) == 0)
1364                         continue;       /* not allowed */
1365
1366                 if (itype == INTR_MSIX)
1367                         navail = pci_msix_count(sc->dev);
1368                 else if (itype == INTR_MSI)
1369                         navail = pci_msi_count(sc->dev);
1370                 else
1371                         navail = 1;
1372 restart:
1373                 if (navail == 0)
1374                         continue;
1375
1376                 iaq->intr_type = itype;
1377                 iaq->intr_flags = 0;
1378
1379                 /*
1380                  * Best option: an interrupt vector for errors, one for the
1381                  * firmware event queue, and one each for each rxq (NIC as well
1382                  * as offload).
1383                  */
1384                 iaq->nirq = T4_EXTRA_INTR;
1385                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1386                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1387                 if (iaq->nirq <= navail &&
1388                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1389                         iaq->intr_flags |= INTR_DIRECT;
1390                         goto allocate;
1391                 }
1392
1393                 /*
1394                  * Second best option: an interrupt vector for errors, one for
1395                  * the firmware event queue, and one each for either NIC or
1396                  * offload rxq's.
1397                  */
1398                 iaq->nirq = T4_EXTRA_INTR;
1399                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1400                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1401                 if (iaq->nirq <= navail &&
1402                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1403                         goto allocate;
1404
1405                 /*
1406                  * Next best option: an interrupt vector for errors, one for the
1407                  * firmware event queue, and at least one per port.  At this
1408                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1409                  * what's available to us.
1410                  */
1411                 iaq->nirq = T4_EXTRA_INTR;
1412                 iaq->nirq += n10g + n1g;
1413                 if (iaq->nirq <= navail) {
1414                         int leftover = navail - iaq->nirq;
1415
1416                         if (n10g > 0) {
1417                                 int target = max(nrxq10g, nofldrxq10g);
1418
1419                                 n = 1;
1420                                 while (n < target && leftover >= n10g) {
1421                                         leftover -= n10g;
1422                                         iaq->nirq += n10g;
1423                                         n++;
1424                                 }
1425                                 iaq->nrxq10g = min(n, nrxq10g);
1426 #ifdef TCP_OFFLOAD
1427                                 if (is_offload(sc))
1428                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1429 #endif
1430                         }
1431
1432                         if (n1g > 0) {
1433                                 int target = max(nrxq1g, nofldrxq1g);
1434
1435                                 n = 1;
1436                                 while (n < target && leftover >= n1g) {
1437                                         leftover -= n1g;
1438                                         iaq->nirq += n1g;
1439                                         n++;
1440                                 }
1441                                 iaq->nrxq1g = min(n, nrxq1g);
1442 #ifdef TCP_OFFLOAD
1443                                 if (is_offload(sc))
1444                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1445 #endif
1446                         }
1447
1448                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1449                                 goto allocate;
1450                 }
1451
1452                 /*
1453                  * Least desirable option: one interrupt vector for everything.
1454                  */
1455                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1456 #ifdef TCP_OFFLOAD
1457                 if (is_offload(sc))
1458                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1459 #endif
1460
1461 allocate:
1462                 navail = iaq->nirq;
1463                 rc = 0;
1464                 if (itype == INTR_MSIX)
1465                         rc = pci_alloc_msix(sc->dev, &navail);
1466                 else if (itype == INTR_MSI)
1467                         rc = pci_alloc_msi(sc->dev, &navail);
1468
1469                 if (rc == 0) {
1470                         if (navail == iaq->nirq)
1471                                 return (0);
1472
1473                         /*
1474                          * Didn't get the number requested.  Use whatever number
1475                          * the kernel is willing to allocate (it's in navail).
1476                          */
1477                         device_printf(sc->dev, "fewer vectors than requested, "
1478                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1479                             itype, iaq->nirq, navail);
1480                         pci_release_msi(sc->dev);
1481                         goto restart;
1482                 }
1483
1484                 device_printf(sc->dev,
1485                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1486                     itype, rc, iaq->nirq, navail);
1487         }
1488
1489         device_printf(sc->dev,
1490             "failed to find a usable interrupt type.  "
1491             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1492             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1493
1494         return (ENXIO);
1495 }
1496
1497 /*
1498  * Install a compatible firmware (if required), establish contact with it (by
1499  * saying hello), and reset the device.  If we end up as the master driver,
1500  * partition adapter resources by providing a configuration file to the
1501  * firmware.
1502  */
1503 static int
1504 prep_firmware(struct adapter *sc)
1505 {
1506         const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1507         int rc;
1508         enum dev_state state;
1509
1510         default_cfg = firmware_get(T4_CFGNAME);
1511
1512         /* Check firmware version and install a different one if necessary */
1513         rc = t4_check_fw_version(sc);
1514         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1515             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1516             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1517             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1518             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1519         if (rc != 0) {
1520                 uint32_t v = 0;
1521
1522                 fw = firmware_get(T4_FWNAME);
1523                 if (fw != NULL) {
1524                         const struct fw_hdr *hdr = (const void *)fw->data;
1525
1526                         v = ntohl(hdr->fw_ver);
1527
1528                         /*
1529                          * The firmware module will not be used if it isn't the
1530                          * same major version as what the driver was compiled
1531                          * with.
1532                          */
1533                         if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
1534                                 device_printf(sc->dev,
1535                                     "Found firmware image but version %d "
1536                                     "can not be used with this driver (%d)\n",
1537                                     G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR);
1538
1539                                 firmware_put(fw, FIRMWARE_UNLOAD);
1540                                 fw = NULL;
1541                         }
1542                 }
1543
1544                 if (fw == NULL && rc < 0) {
1545                         device_printf(sc->dev, "No usable firmware. "
1546                             "card has %d.%d.%d, driver compiled with %d.%d.%d",
1547                             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1548                             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1549                             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1550                             FW_VERSION_MAJOR, FW_VERSION_MINOR,
1551                             FW_VERSION_MICRO);
1552                         rc = EAGAIN;
1553                         goto done;
1554                 }
1555
1556                 /*
1557                  * Always upgrade, even for minor/micro/build mismatches.
1558                  * Downgrade only for a major version mismatch or if
1559                  * force_firmware_install was specified.
1560                  */
1561                 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
1562                         device_printf(sc->dev,
1563                             "installing firmware %d.%d.%d.%d on card.\n",
1564                             G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1565                             G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1566
1567                         rc = -t4_load_fw(sc, fw->data, fw->datasize);
1568                         if (rc != 0) {
1569                                 device_printf(sc->dev,
1570                                     "failed to install firmware: %d\n", rc);
1571                                 goto done;
1572                         } else {
1573                                 /* refresh */
1574                                 (void) t4_check_fw_version(sc);
1575                                 snprintf(sc->fw_version,
1576                                     sizeof(sc->fw_version), "%u.%u.%u.%u",
1577                                     G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1578                                     G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1579                                     G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1580                                     G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1581                         }
1582                 }
1583         }
1584
1585         /* Contact firmware.  */
1586         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1587         if (rc < 0) {
1588                 rc = -rc;
1589                 device_printf(sc->dev,
1590                     "failed to connect to the firmware: %d.\n", rc);
1591                 goto done;
1592         }
1593         if (rc == sc->mbox)
1594                 sc->flags |= MASTER_PF;
1595
1596         /* Reset device */
1597         rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1598         if (rc != 0) {
1599                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1600                 if (rc != ETIMEDOUT && rc != EIO)
1601                         t4_fw_bye(sc, sc->mbox);
1602                 goto done;
1603         }
1604
1605         /* Partition adapter resources as specified in the config file. */
1606         if (sc->flags & MASTER_PF) {
1607                 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
1608                         char s[32];
1609
1610                         snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
1611                         cfg = firmware_get(s);
1612                         if (cfg == NULL) {
1613                                 device_printf(sc->dev,
1614                                     "unable to locate %s module, "
1615                                     "will use default config file.\n", s);
1616                         }
1617                 }
1618
1619                 rc = partition_resources(sc, cfg ? cfg : default_cfg);
1620                 if (rc != 0)
1621                         goto done;      /* error message displayed already */
1622         }
1623
1624         sc->flags |= FW_OK;
1625
1626 done:
1627         if (fw != NULL)
1628                 firmware_put(fw, FIRMWARE_UNLOAD);
1629         if (cfg != NULL)
1630                 firmware_put(cfg, FIRMWARE_UNLOAD);
1631         if (default_cfg != NULL)
1632                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
1633
1634         return (rc);
1635 }
1636
1637 #define FW_PARAM_DEV(param) \
1638         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1639          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1640 #define FW_PARAM_PFVF(param) \
1641         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1642          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1643
1644 /*
1645  * Upload configuration file to card's memory.
1646  */
1647 static int
1648 upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1649     uint32_t *ma)
1650 {
1651         int rc, i;
1652         uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1653         const uint32_t *b;
1654
1655         /* Figure out where the firmware wants us to upload it. */
1656         param = FW_PARAM_DEV(CF);
1657         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1658         if (rc != 0) {
1659                 /* Firmwares without config file support will fail this way */
1660                 device_printf(sc->dev,
1661                     "failed to query config file location: %d.\n", rc);
1662                 return (rc);
1663         }
1664         *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1665         *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1666
1667         if (maddr & 3) {
1668                 device_printf(sc->dev,
1669                     "cannot upload config file (type %u, addr %x).\n",
1670                     mtype, maddr);
1671                 return (EFAULT);
1672         }
1673
1674         /* Translate mtype/maddr to an address suitable for the PCIe window */
1675         val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1676         val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1677         switch (mtype) {
1678         case FW_MEMTYPE_CF_EDC0:
1679                 if (!(val & F_EDRAM0_ENABLE))
1680                         goto err;
1681                 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1682                 maddr += G_EDRAM0_BASE(bar) << 20;
1683                 break;
1684
1685         case FW_MEMTYPE_CF_EDC1:
1686                 if (!(val & F_EDRAM1_ENABLE))
1687                         goto err;
1688                 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1689                 maddr += G_EDRAM1_BASE(bar) << 20;
1690                 break;
1691
1692         case FW_MEMTYPE_CF_EXTMEM:
1693                 if (!(val & F_EXT_MEM_ENABLE))
1694                         goto err;
1695                 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1696                 maddr += G_EXT_MEM_BASE(bar) << 20;
1697                 break;
1698
1699         default:
1700 err:
1701                 device_printf(sc->dev,
1702                     "cannot upload config file (type %u, enabled %u).\n",
1703                     mtype, val);
1704                 return (EFAULT);
1705         }
1706
1707         /*
1708          * Position the PCIe window (we use memwin2) to the 16B aligned area
1709          * just at/before the upload location.
1710          */
1711         win = maddr & ~0xf;
1712         off = maddr - win;  /* offset from the start of the window. */
1713         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1714         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1715
1716         remaining = fw->datasize;
1717         if (remaining > FLASH_CFG_MAX_SIZE ||
1718             remaining > MEMWIN2_APERTURE - off) {
1719                 device_printf(sc->dev, "cannot upload config file all at once "
1720                     "(size %u, max %u, room %u).\n",
1721                     remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1722                 return (EFBIG);
1723         }
1724
1725         /*
1726          * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1727          * stuffing/comments at the end of the config file so it's ok to simply
1728          * throw away the last remaining bytes when the config file is not an
1729          * exact multiple of 4.
1730          */
1731         b = fw->data;
1732         for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1733                 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1734
1735         return (rc);
1736 }
1737
1738 /*
1739  * Partition chip resources for use between various PFs, VFs, etc.  This is done
1740  * by uploading the firmware configuration file to the adapter and instructing
1741  * the firmware to process it.
1742  */
1743 static int
1744 partition_resources(struct adapter *sc, const struct firmware *cfg)
1745 {
1746         int rc;
1747         struct fw_caps_config_cmd caps;
1748         uint32_t mtype, maddr, finicsum, cfcsum;
1749
1750         rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1751         if (rc != 0) {
1752                 mtype = FW_MEMTYPE_CF_FLASH;
1753                 maddr = t4_flash_cfg_addr(sc);
1754         }
1755
1756         bzero(&caps, sizeof(caps));
1757         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1758             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1759         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1760             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1761             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1762         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1763         if (rc != 0) {
1764                 device_printf(sc->dev,
1765                     "failed to pre-process config file: %d.\n", rc);
1766                 return (rc);
1767         }
1768
1769         finicsum = be32toh(caps.finicsum);
1770         cfcsum = be32toh(caps.cfcsum);
1771         if (finicsum != cfcsum) {
1772                 device_printf(sc->dev,
1773                     "WARNING: config file checksum mismatch: %08x %08x\n",
1774                     finicsum, cfcsum);
1775         }
1776         sc->cfcsum = cfcsum;
1777
1778 #define LIMIT_CAPS(x) do { \
1779         caps.x &= htobe16(t4_##x##_allowed); \
1780         sc->x = htobe16(caps.x); \
1781 } while (0)
1782
1783         /*
1784          * Let the firmware know what features will (not) be used so it can tune
1785          * things accordingly.
1786          */
1787         LIMIT_CAPS(linkcaps);
1788         LIMIT_CAPS(niccaps);
1789         LIMIT_CAPS(toecaps);
1790         LIMIT_CAPS(rdmacaps);
1791         LIMIT_CAPS(iscsicaps);
1792         LIMIT_CAPS(fcoecaps);
1793 #undef LIMIT_CAPS
1794
1795         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1796             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1797         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1798         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1799         if (rc != 0) {
1800                 device_printf(sc->dev,
1801                     "failed to process config file: %d.\n", rc);
1802                 return (rc);
1803         }
1804
1805         return (0);
1806 }
1807
1808 /*
1809  * Retrieve parameters that are needed (or nice to have) prior to calling
1810  * t4_sge_init and t4_fw_initialize.
1811  */
1812 static int
1813 get_params__pre_init(struct adapter *sc)
1814 {
1815         int rc;
1816         uint32_t param[2], val[2];
1817         struct fw_devlog_cmd cmd;
1818         struct devlog_params *dlog = &sc->params.devlog;
1819
1820         param[0] = FW_PARAM_DEV(PORTVEC);
1821         param[1] = FW_PARAM_DEV(CCLK);
1822         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1823         if (rc != 0) {
1824                 device_printf(sc->dev,
1825                     "failed to query parameters (pre_init): %d.\n", rc);
1826                 return (rc);
1827         }
1828
1829         sc->params.portvec = val[0];
1830         sc->params.nports = bitcount32(val[0]);
1831         sc->params.vpd.cclk = val[1];
1832
1833         /* Read device log parameters. */
1834         bzero(&cmd, sizeof(cmd));
1835         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1836             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1837         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1838         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1839         if (rc != 0) {
1840                 device_printf(sc->dev,
1841                     "failed to get devlog parameters: %d.\n", rc);
1842                 bzero(dlog, sizeof (*dlog));
1843                 rc = 0; /* devlog isn't critical for device operation */
1844         } else {
1845                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1846                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1847                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1848                 dlog->size = be32toh(cmd.memsize_devlog);
1849         }
1850
1851         return (rc);
1852 }
1853
1854 /*
1855  * Retrieve various parameters that are of interest to the driver.  The device
1856  * has been initialized by the firmware at this point.
1857  */
1858 static int
1859 get_params__post_init(struct adapter *sc)
1860 {
1861         int rc;
1862         uint32_t param[7], val[7];
1863         struct fw_caps_config_cmd caps;
1864
1865         param[0] = FW_PARAM_PFVF(IQFLINT_START);
1866         param[1] = FW_PARAM_PFVF(EQ_START);
1867         param[2] = FW_PARAM_PFVF(FILTER_START);
1868         param[3] = FW_PARAM_PFVF(FILTER_END);
1869         param[4] = FW_PARAM_PFVF(L2T_START);
1870         param[5] = FW_PARAM_PFVF(L2T_END);
1871         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1872         if (rc != 0) {
1873                 device_printf(sc->dev,
1874                     "failed to query parameters (post_init): %d.\n", rc);
1875                 return (rc);
1876         }
1877
1878         sc->sge.iq_start = val[0];
1879         sc->sge.eq_start = val[1];
1880         sc->tids.ftid_base = val[2];
1881         sc->tids.nftids = val[3] - val[2] + 1;
1882         sc->vres.l2t.start = val[4];
1883         sc->vres.l2t.size = val[5] - val[4] + 1;
1884         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
1885             ("%s: L2 table size (%u) larger than expected (%u)",
1886             __func__, sc->vres.l2t.size, L2T_SIZE));
1887
1888         /* get capabilites */
1889         bzero(&caps, sizeof(caps));
1890         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1891             F_FW_CMD_REQUEST | F_FW_CMD_READ);
1892         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1893         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1894         if (rc != 0) {
1895                 device_printf(sc->dev,
1896                     "failed to get card capabilities: %d.\n", rc);
1897                 return (rc);
1898         }
1899
1900         if (caps.toecaps) {
1901                 /* query offload-related parameters */
1902                 param[0] = FW_PARAM_DEV(NTID);
1903                 param[1] = FW_PARAM_PFVF(SERVER_START);
1904                 param[2] = FW_PARAM_PFVF(SERVER_END);
1905                 param[3] = FW_PARAM_PFVF(TDDP_START);
1906                 param[4] = FW_PARAM_PFVF(TDDP_END);
1907                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1908                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1909                 if (rc != 0) {
1910                         device_printf(sc->dev,
1911                             "failed to query TOE parameters: %d.\n", rc);
1912                         return (rc);
1913                 }
1914                 sc->tids.ntids = val[0];
1915                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1916                 sc->tids.stid_base = val[1];
1917                 sc->tids.nstids = val[2] - val[1] + 1;
1918                 sc->vres.ddp.start = val[3];
1919                 sc->vres.ddp.size = val[4] - val[3] + 1;
1920                 sc->params.ofldq_wr_cred = val[5];
1921                 sc->params.offload = 1;
1922         }
1923         if (caps.rdmacaps) {
1924                 param[0] = FW_PARAM_PFVF(STAG_START);
1925                 param[1] = FW_PARAM_PFVF(STAG_END);
1926                 param[2] = FW_PARAM_PFVF(RQ_START);
1927                 param[3] = FW_PARAM_PFVF(RQ_END);
1928                 param[4] = FW_PARAM_PFVF(PBL_START);
1929                 param[5] = FW_PARAM_PFVF(PBL_END);
1930                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1931                 if (rc != 0) {
1932                         device_printf(sc->dev,
1933                             "failed to query RDMA parameters(1): %d.\n", rc);
1934                         return (rc);
1935                 }
1936                 sc->vres.stag.start = val[0];
1937                 sc->vres.stag.size = val[1] - val[0] + 1;
1938                 sc->vres.rq.start = val[2];
1939                 sc->vres.rq.size = val[3] - val[2] + 1;
1940                 sc->vres.pbl.start = val[4];
1941                 sc->vres.pbl.size = val[5] - val[4] + 1;
1942
1943                 param[0] = FW_PARAM_PFVF(SQRQ_START);
1944                 param[1] = FW_PARAM_PFVF(SQRQ_END);
1945                 param[2] = FW_PARAM_PFVF(CQ_START);
1946                 param[3] = FW_PARAM_PFVF(CQ_END);
1947                 param[4] = FW_PARAM_PFVF(OCQ_START);
1948                 param[5] = FW_PARAM_PFVF(OCQ_END);
1949                 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
1950                 if (rc != 0) {
1951                         device_printf(sc->dev,
1952                             "failed to query RDMA parameters(2): %d.\n", rc);
1953                         return (rc);
1954                 }
1955                 sc->vres.qp.start = val[0];
1956                 sc->vres.qp.size = val[1] - val[0] + 1;
1957                 sc->vres.cq.start = val[2];
1958                 sc->vres.cq.size = val[3] - val[2] + 1;
1959                 sc->vres.ocq.start = val[4];
1960                 sc->vres.ocq.size = val[5] - val[4] + 1;
1961         }
1962         if (caps.iscsicaps) {
1963                 param[0] = FW_PARAM_PFVF(ISCSI_START);
1964                 param[1] = FW_PARAM_PFVF(ISCSI_END);
1965                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1966                 if (rc != 0) {
1967                         device_printf(sc->dev,
1968                             "failed to query iSCSI parameters: %d.\n", rc);
1969                         return (rc);
1970                 }
1971                 sc->vres.iscsi.start = val[0];
1972                 sc->vres.iscsi.size = val[1] - val[0] + 1;
1973         }
1974
1975         /* These are finalized by FW initialization, load their values now */
1976         val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1977         sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1978         sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1979         t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1980
1981         return (rc);
1982 }
1983
1984 #undef FW_PARAM_PFVF
1985 #undef FW_PARAM_DEV
1986
1987 static void
1988 t4_set_desc(struct adapter *sc)
1989 {
1990         char buf[128];
1991         struct adapter_params *p = &sc->params;
1992
1993         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
1994             p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
1995
1996         device_set_desc_copy(sc->dev, buf);
1997 }
1998
1999 static void
2000 build_medialist(struct port_info *pi)
2001 {
2002         struct ifmedia *media = &pi->media;
2003         int data, m;
2004
2005         PORT_LOCK(pi);
2006
2007         ifmedia_removeall(media);
2008
2009         m = IFM_ETHER | IFM_FDX;
2010         data = (pi->port_type << 8) | pi->mod_type;
2011
2012         switch(pi->port_type) {
2013         case FW_PORT_TYPE_BT_XFI:
2014                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2015                 break;
2016
2017         case FW_PORT_TYPE_BT_XAUI:
2018                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2019                 /* fall through */
2020
2021         case FW_PORT_TYPE_BT_SGMII:
2022                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2023                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2024                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2025                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2026                 break;
2027
2028         case FW_PORT_TYPE_CX4:
2029                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2030                 ifmedia_set(media, m | IFM_10G_CX4);
2031                 break;
2032
2033         case FW_PORT_TYPE_SFP:
2034         case FW_PORT_TYPE_FIBER_XFI:
2035         case FW_PORT_TYPE_FIBER_XAUI:
2036                 switch (pi->mod_type) {
2037
2038                 case FW_PORT_MOD_TYPE_LR:
2039                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2040                         ifmedia_set(media, m | IFM_10G_LR);
2041                         break;
2042
2043                 case FW_PORT_MOD_TYPE_SR:
2044                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2045                         ifmedia_set(media, m | IFM_10G_SR);
2046                         break;
2047
2048                 case FW_PORT_MOD_TYPE_LRM:
2049                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2050                         ifmedia_set(media, m | IFM_10G_LRM);
2051                         break;
2052
2053                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2054                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2055                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2056                         ifmedia_set(media, m | IFM_10G_TWINAX);
2057                         break;
2058
2059                 case FW_PORT_MOD_TYPE_NONE:
2060                         m &= ~IFM_FDX;
2061                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2062                         ifmedia_set(media, m | IFM_NONE);
2063                         break;
2064
2065                 case FW_PORT_MOD_TYPE_NA:
2066                 case FW_PORT_MOD_TYPE_ER:
2067                 default:
2068                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2069                         ifmedia_set(media, m | IFM_UNKNOWN);
2070                         break;
2071                 }
2072                 break;
2073
2074         case FW_PORT_TYPE_KX4:
2075         case FW_PORT_TYPE_KX:
2076         case FW_PORT_TYPE_KR:
2077         default:
2078                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2079                 ifmedia_set(media, m | IFM_UNKNOWN);
2080                 break;
2081         }
2082
2083         PORT_UNLOCK(pi);
2084 }
2085
2086 #define FW_MAC_EXACT_CHUNK      7
2087
2088 /*
2089  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2090  * indicates which parameters should be programmed (the rest are left alone).
2091  */
2092 static int
2093 update_mac_settings(struct port_info *pi, int flags)
2094 {
2095         int rc;
2096         struct ifnet *ifp = pi->ifp;
2097         struct adapter *sc = pi->adapter;
2098         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2099
2100         ASSERT_SYNCHRONIZED_OP(sc);
2101         KASSERT(flags, ("%s: not told what to update.", __func__));
2102
2103         if (flags & XGMAC_MTU)
2104                 mtu = ifp->if_mtu;
2105
2106         if (flags & XGMAC_PROMISC)
2107                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2108
2109         if (flags & XGMAC_ALLMULTI)
2110                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2111
2112         if (flags & XGMAC_VLANEX)
2113                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2114
2115         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2116             vlanex, false);
2117         if (rc) {
2118                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2119                 return (rc);
2120         }
2121
2122         if (flags & XGMAC_UCADDR) {
2123                 uint8_t ucaddr[ETHER_ADDR_LEN];
2124
2125                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2126                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2127                     ucaddr, true, true);
2128                 if (rc < 0) {
2129                         rc = -rc;
2130                         if_printf(ifp, "change_mac failed: %d\n", rc);
2131                         return (rc);
2132                 } else {
2133                         pi->xact_addr_filt = rc;
2134                         rc = 0;
2135                 }
2136         }
2137
2138         if (flags & XGMAC_MCADDRS) {
2139                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2140                 int del = 1;
2141                 uint64_t hash = 0;
2142                 struct ifmultiaddr *ifma;
2143                 int i = 0, j;
2144
2145                 if_maddr_rlock(ifp);
2146                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2147                         if (ifma->ifma_addr->sa_family != AF_LINK)
2148                                 continue;
2149                         mcaddr[i++] =
2150                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2151
2152                         if (i == FW_MAC_EXACT_CHUNK) {
2153                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2154                                     del, i, mcaddr, NULL, &hash, 0);
2155                                 if (rc < 0) {
2156                                         rc = -rc;
2157                                         for (j = 0; j < i; j++) {
2158                                                 if_printf(ifp,
2159                                                     "failed to add mc address"
2160                                                     " %02x:%02x:%02x:"
2161                                                     "%02x:%02x:%02x rc=%d\n",
2162                                                     mcaddr[j][0], mcaddr[j][1],
2163                                                     mcaddr[j][2], mcaddr[j][3],
2164                                                     mcaddr[j][4], mcaddr[j][5],
2165                                                     rc);
2166                                         }
2167                                         goto mcfail;
2168                                 }
2169                                 del = 0;
2170                                 i = 0;
2171                         }
2172                 }
2173                 if (i > 0) {
2174                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2175                             del, i, mcaddr, NULL, &hash, 0);
2176                         if (rc < 0) {
2177                                 rc = -rc;
2178                                 for (j = 0; j < i; j++) {
2179                                         if_printf(ifp,
2180                                             "failed to add mc address"
2181                                             " %02x:%02x:%02x:"
2182                                             "%02x:%02x:%02x rc=%d\n",
2183                                             mcaddr[j][0], mcaddr[j][1],
2184                                             mcaddr[j][2], mcaddr[j][3],
2185                                             mcaddr[j][4], mcaddr[j][5],
2186                                             rc);
2187                                 }
2188                                 goto mcfail;
2189                         }
2190                 }
2191
2192                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2193                 if (rc != 0)
2194                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2195 mcfail:
2196                 if_maddr_runlock(ifp);
2197         }
2198
2199         return (rc);
2200 }
2201
2202 int
2203 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2204     char *wmesg)
2205 {
2206         int rc, pri;
2207
2208 #ifdef WITNESS
2209         /* the caller thinks it's ok to sleep, but is it really? */
2210         if (flags & SLEEP_OK)
2211                 pause("t4slptst", 1);
2212 #endif
2213
2214         if (INTR_OK)
2215                 pri = PCATCH;
2216         else
2217                 pri = 0;
2218
2219         ADAPTER_LOCK(sc);
2220         for (;;) {
2221
2222                 if (pi && IS_DOOMED(pi)) {
2223                         rc = ENXIO;
2224                         goto done;
2225                 }
2226
2227                 if (!IS_BUSY(sc)) {
2228                         rc = 0;
2229                         break;
2230                 }
2231
2232                 if (!(flags & SLEEP_OK)) {
2233                         rc = EBUSY;
2234                         goto done;
2235                 }
2236
2237                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2238                         rc = EINTR;
2239                         goto done;
2240                 }
2241         }
2242
2243         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2244         SET_BUSY(sc);
2245 #ifdef INVARIANTS
2246         sc->last_op = wmesg;
2247         sc->last_op_thr = curthread;
2248 #endif
2249
2250 done:
2251         if (!(flags & HOLD_LOCK) || rc)
2252                 ADAPTER_UNLOCK(sc);
2253
2254         return (rc);
2255 }
2256
2257 void
2258 end_synchronized_op(struct adapter *sc, int flags)
2259 {
2260
2261         if (flags & LOCK_HELD)
2262                 ADAPTER_LOCK_ASSERT_OWNED(sc);
2263         else
2264                 ADAPTER_LOCK(sc);
2265
2266         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2267         CLR_BUSY(sc);
2268         wakeup(&sc->flags);
2269         ADAPTER_UNLOCK(sc);
2270 }
2271
2272 static int
2273 cxgbe_init_synchronized(struct port_info *pi)
2274 {
2275         struct adapter *sc = pi->adapter;
2276         struct ifnet *ifp = pi->ifp;
2277         int rc = 0;
2278
2279         ASSERT_SYNCHRONIZED_OP(sc);
2280
2281         if (isset(&sc->open_device_map, pi->port_id)) {
2282                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2283                     ("mismatch between open_device_map and if_drv_flags"));
2284                 return (0);     /* already running */
2285         }
2286
2287         if (!(sc->flags & FULL_INIT_DONE) &&
2288             ((rc = adapter_full_init(sc)) != 0))
2289                 return (rc);    /* error message displayed already */
2290
2291         if (!(pi->flags & PORT_INIT_DONE) &&
2292             ((rc = port_full_init(pi)) != 0))
2293                 return (rc); /* error message displayed already */
2294
2295         rc = update_mac_settings(pi, XGMAC_ALL);
2296         if (rc)
2297                 goto done;      /* error message displayed already */
2298
2299         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2300         if (rc != 0) {
2301                 if_printf(ifp, "start_link failed: %d\n", rc);
2302                 goto done;
2303         }
2304
2305         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2306         if (rc != 0) {
2307                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2308                 goto done;
2309         }
2310
2311         /* all ok */
2312         setbit(&sc->open_device_map, pi->port_id);
2313         PORT_LOCK(pi);
2314         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2315         PORT_UNLOCK(pi);
2316
2317         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2318 done:
2319         if (rc != 0)
2320                 cxgbe_uninit_synchronized(pi);
2321
2322         return (rc);
2323 }
2324
2325 /*
2326  * Idempotent.
2327  */
2328 static int
2329 cxgbe_uninit_synchronized(struct port_info *pi)
2330 {
2331         struct adapter *sc = pi->adapter;
2332         struct ifnet *ifp = pi->ifp;
2333         int rc;
2334
2335         ASSERT_SYNCHRONIZED_OP(sc);
2336
2337         /*
2338          * Disable the VI so that all its data in either direction is discarded
2339          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2340          * tick) intact as the TP can deliver negative advice or data that it's
2341          * holding in its RAM (for an offloaded connection) even after the VI is
2342          * disabled.
2343          */
2344         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2345         if (rc) {
2346                 if_printf(ifp, "disable_vi failed: %d\n", rc);
2347                 return (rc);
2348         }
2349
2350         clrbit(&sc->open_device_map, pi->port_id);
2351         PORT_LOCK(pi);
2352         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2353         PORT_UNLOCK(pi);
2354
2355         pi->link_cfg.link_ok = 0;
2356         pi->link_cfg.speed = 0;
2357         t4_os_link_changed(sc, pi->port_id, 0);
2358
2359         return (0);
2360 }
2361
2362 /*
2363  * It is ok for this function to fail midway and return right away.  t4_detach
2364  * will walk the entire sc->irq list and clean up whatever is valid.
2365  */
2366 static int
2367 setup_intr_handlers(struct adapter *sc)
2368 {
2369         int rc, rid, p, q;
2370         char s[8];
2371         struct irq *irq;
2372         struct port_info *pi;
2373         struct sge_rxq *rxq;
2374 #ifdef TCP_OFFLOAD
2375         struct sge_ofld_rxq *ofld_rxq;
2376 #endif
2377
2378         /*
2379          * Setup interrupts.
2380          */
2381         irq = &sc->irq[0];
2382         rid = sc->intr_type == INTR_INTX ? 0 : 1;
2383         if (sc->intr_count == 1) {
2384                 KASSERT(!(sc->flags & INTR_DIRECT),
2385                     ("%s: single interrupt && INTR_DIRECT?", __func__));
2386
2387                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
2388                 if (rc != 0)
2389                         return (rc);
2390         } else {
2391                 /* Multiple interrupts. */
2392                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2393                     ("%s: too few intr.", __func__));
2394
2395                 /* The first one is always error intr */
2396                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
2397                 if (rc != 0)
2398                         return (rc);
2399                 irq++;
2400                 rid++;
2401
2402                 /* The second one is always the firmware event queue */
2403                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
2404                     "evt");
2405                 if (rc != 0)
2406                         return (rc);
2407                 irq++;
2408                 rid++;
2409
2410                 /*
2411                  * Note that if INTR_DIRECT is not set then either the NIC rx
2412                  * queues or (exclusive or) the TOE rx queueus will be taking
2413                  * direct interrupts.
2414                  *
2415                  * There is no need to check for is_offload(sc) as nofldrxq
2416                  * will be 0 if offload is disabled.
2417                  */
2418                 for_each_port(sc, p) {
2419                         pi = sc->port[p];
2420
2421 #ifdef TCP_OFFLOAD
2422                         /*
2423                          * Skip over the NIC queues if they aren't taking direct
2424                          * interrupts.
2425                          */
2426                         if (!(sc->flags & INTR_DIRECT) &&
2427                             pi->nofldrxq > pi->nrxq)
2428                                 goto ofld_queues;
2429 #endif
2430                         rxq = &sc->sge.rxq[pi->first_rxq];
2431                         for (q = 0; q < pi->nrxq; q++, rxq++) {
2432                                 snprintf(s, sizeof(s), "%d.%d", p, q);
2433                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
2434                                     s);
2435                                 if (rc != 0)
2436                                         return (rc);
2437                                 irq++;
2438                                 rid++;
2439                         }
2440
2441 #ifdef TCP_OFFLOAD
2442                         /*
2443                          * Skip over the offload queues if they aren't taking
2444                          * direct interrupts.
2445                          */
2446                         if (!(sc->flags & INTR_DIRECT))
2447                                 continue;
2448 ofld_queues:
2449                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2450                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2451                                 snprintf(s, sizeof(s), "%d,%d", p, q);
2452                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
2453                                     ofld_rxq, s);
2454                                 if (rc != 0)
2455                                         return (rc);
2456                                 irq++;
2457                                 rid++;
2458                         }
2459 #endif
2460                 }
2461         }
2462
2463         return (0);
2464 }
2465
2466 static int
2467 adapter_full_init(struct adapter *sc)
2468 {
2469         int rc, i;
2470
2471         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2472         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2473             ("%s: FULL_INIT_DONE already", __func__));
2474
2475         /*
2476          * queues that belong to the adapter (not any particular port).
2477          */
2478         rc = t4_setup_adapter_queues(sc);
2479         if (rc != 0)
2480                 goto done;
2481
2482         for (i = 0; i < nitems(sc->tq); i++) {
2483                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2484                     taskqueue_thread_enqueue, &sc->tq[i]);
2485                 if (sc->tq[i] == NULL) {
2486                         device_printf(sc->dev,
2487                             "failed to allocate task queue %d\n", i);
2488                         rc = ENOMEM;
2489                         goto done;
2490                 }
2491                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2492                     device_get_nameunit(sc->dev), i);
2493         }
2494
2495         t4_intr_enable(sc);
2496         sc->flags |= FULL_INIT_DONE;
2497 done:
2498         if (rc != 0)
2499                 adapter_full_uninit(sc);
2500
2501         return (rc);
2502 }
2503
2504 static int
2505 adapter_full_uninit(struct adapter *sc)
2506 {
2507         int i;
2508
2509         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2510
2511         t4_teardown_adapter_queues(sc);
2512
2513         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
2514                 taskqueue_free(sc->tq[i]);
2515                 sc->tq[i] = NULL;
2516         }
2517
2518         sc->flags &= ~FULL_INIT_DONE;
2519
2520         return (0);
2521 }
2522
2523 static int
2524 port_full_init(struct port_info *pi)
2525 {
2526         struct adapter *sc = pi->adapter;
2527         struct ifnet *ifp = pi->ifp;
2528         uint16_t *rss;
2529         struct sge_rxq *rxq;
2530         int rc, i;
2531
2532         ASSERT_SYNCHRONIZED_OP(sc);
2533         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2534             ("%s: PORT_INIT_DONE already", __func__));
2535
2536         sysctl_ctx_init(&pi->ctx);
2537         pi->flags |= PORT_SYSCTL_CTX;
2538
2539         /*
2540          * Allocate tx/rx/fl queues for this port.
2541          */
2542         rc = t4_setup_port_queues(pi);
2543         if (rc != 0)
2544                 goto done;      /* error message displayed already */
2545
2546         /*
2547          * Setup RSS for this port.
2548          */
2549         rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2550             M_ZERO | M_WAITOK);
2551         for_each_rxq(pi, i, rxq) {
2552                 rss[i] = rxq->iq.abs_id;
2553         }
2554         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2555             pi->rss_size, rss, pi->nrxq);
2556         free(rss, M_CXGBE);
2557         if (rc != 0) {
2558                 if_printf(ifp, "rss_config failed: %d\n", rc);
2559                 goto done;
2560         }
2561
2562         pi->flags |= PORT_INIT_DONE;
2563 done:
2564         if (rc != 0)
2565                 port_full_uninit(pi);
2566
2567         return (rc);
2568 }
2569
2570 /*
2571  * Idempotent.
2572  */
2573 static int
2574 port_full_uninit(struct port_info *pi)
2575 {
2576         struct adapter *sc = pi->adapter;
2577         int i;
2578         struct sge_rxq *rxq;
2579         struct sge_txq *txq;
2580 #ifdef TCP_OFFLOAD
2581         struct sge_ofld_rxq *ofld_rxq;
2582         struct sge_wrq *ofld_txq;
2583 #endif
2584
2585         if (pi->flags & PORT_INIT_DONE) {
2586
2587                 /* Need to quiesce queues.  XXX: ctrl queues? */
2588
2589                 for_each_txq(pi, i, txq) {
2590                         quiesce_eq(sc, &txq->eq);
2591                 }
2592
2593 #ifdef TCP_OFFLOAD
2594                 for_each_ofld_txq(pi, i, ofld_txq) {
2595                         quiesce_eq(sc, &ofld_txq->eq);
2596                 }
2597 #endif
2598
2599                 for_each_rxq(pi, i, rxq) {
2600                         quiesce_iq(sc, &rxq->iq);
2601                         quiesce_fl(sc, &rxq->fl);
2602                 }
2603
2604 #ifdef TCP_OFFLOAD
2605                 for_each_ofld_rxq(pi, i, ofld_rxq) {
2606                         quiesce_iq(sc, &ofld_rxq->iq);
2607                         quiesce_fl(sc, &ofld_rxq->fl);
2608                 }
2609 #endif
2610         }
2611
2612         t4_teardown_port_queues(pi);
2613         pi->flags &= ~PORT_INIT_DONE;
2614
2615         return (0);
2616 }
2617
2618 static void
2619 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
2620 {
2621         EQ_LOCK(eq);
2622         eq->flags |= EQ_DOOMED;
2623
2624         /*
2625          * Wait for the response to a credit flush if one's
2626          * pending.
2627          */
2628         while (eq->flags & EQ_CRFLUSHED)
2629                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2630         EQ_UNLOCK(eq);
2631
2632         callout_drain(&eq->tx_callout); /* XXX: iffy */
2633         pause("callout", 10);           /* Still iffy */
2634
2635         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2636 }
2637
2638 static void
2639 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
2640 {
2641         (void) sc;      /* unused */
2642
2643         /* Synchronize with the interrupt handler */
2644         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2645                 pause("iqfree", 1);
2646 }
2647
2648 static void
2649 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
2650 {
2651         mtx_lock(&sc->sfl_lock);
2652         FL_LOCK(fl);
2653         fl->flags |= FL_DOOMED;
2654         FL_UNLOCK(fl);
2655         mtx_unlock(&sc->sfl_lock);
2656
2657         callout_drain(&sc->sfl_callout);
2658         KASSERT((fl->flags & FL_STARVING) == 0,
2659             ("%s: still starving", __func__));
2660 }
2661
2662 static int
2663 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2664     driver_intr_t *handler, void *arg, char *name)
2665 {
2666         int rc;
2667
2668         irq->rid = rid;
2669         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2670             RF_SHAREABLE | RF_ACTIVE);
2671         if (irq->res == NULL) {
2672                 device_printf(sc->dev,
2673                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2674                 return (ENOMEM);
2675         }
2676
2677         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2678             NULL, handler, arg, &irq->tag);
2679         if (rc != 0) {
2680                 device_printf(sc->dev,
2681                     "failed to setup interrupt for rid %d, name %s: %d\n",
2682                     rid, name, rc);
2683         } else if (name)
2684                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2685
2686         return (rc);
2687 }
2688
2689 static int
2690 t4_free_irq(struct adapter *sc, struct irq *irq)
2691 {
2692         if (irq->tag)
2693                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
2694         if (irq->res)
2695                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2696
2697         bzero(irq, sizeof(*irq));
2698
2699         return (0);
2700 }
2701
2702 static void
2703 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2704     unsigned int end)
2705 {
2706         uint32_t *p = (uint32_t *)(buf + start);
2707
2708         for ( ; start <= end; start += sizeof(uint32_t))
2709                 *p++ = t4_read_reg(sc, start);
2710 }
2711
2712 static void
2713 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2714 {
2715         int i;
2716         static const unsigned int reg_ranges[] = {
2717                 0x1008, 0x1108,
2718                 0x1180, 0x11b4,
2719                 0x11fc, 0x123c,
2720                 0x1300, 0x173c,
2721                 0x1800, 0x18fc,
2722                 0x3000, 0x30d8,
2723                 0x30e0, 0x5924,
2724                 0x5960, 0x59d4,
2725                 0x5a00, 0x5af8,
2726                 0x6000, 0x6098,
2727                 0x6100, 0x6150,
2728                 0x6200, 0x6208,
2729                 0x6240, 0x6248,
2730                 0x6280, 0x6338,
2731                 0x6370, 0x638c,
2732                 0x6400, 0x643c,
2733                 0x6500, 0x6524,
2734                 0x6a00, 0x6a38,
2735                 0x6a60, 0x6a78,
2736                 0x6b00, 0x6b84,
2737                 0x6bf0, 0x6c84,
2738                 0x6cf0, 0x6d84,
2739                 0x6df0, 0x6e84,
2740                 0x6ef0, 0x6f84,
2741                 0x6ff0, 0x7084,
2742                 0x70f0, 0x7184,
2743                 0x71f0, 0x7284,
2744                 0x72f0, 0x7384,
2745                 0x73f0, 0x7450,
2746                 0x7500, 0x7530,
2747                 0x7600, 0x761c,
2748                 0x7680, 0x76cc,
2749                 0x7700, 0x7798,
2750                 0x77c0, 0x77fc,
2751                 0x7900, 0x79fc,
2752                 0x7b00, 0x7c38,
2753                 0x7d00, 0x7efc,
2754                 0x8dc0, 0x8e1c,
2755                 0x8e30, 0x8e78,
2756                 0x8ea0, 0x8f6c,
2757                 0x8fc0, 0x9074,
2758                 0x90fc, 0x90fc,
2759                 0x9400, 0x9458,
2760                 0x9600, 0x96bc,
2761                 0x9800, 0x9808,
2762                 0x9820, 0x983c,
2763                 0x9850, 0x9864,
2764                 0x9c00, 0x9c6c,
2765                 0x9c80, 0x9cec,
2766                 0x9d00, 0x9d6c,
2767                 0x9d80, 0x9dec,
2768                 0x9e00, 0x9e6c,
2769                 0x9e80, 0x9eec,
2770                 0x9f00, 0x9f6c,
2771                 0x9f80, 0x9fec,
2772                 0xd004, 0xd03c,
2773                 0xdfc0, 0xdfe0,
2774                 0xe000, 0xea7c,
2775                 0xf000, 0x11190,
2776                 0x19040, 0x1906c,
2777                 0x19078, 0x19080,
2778                 0x1908c, 0x19124,
2779                 0x19150, 0x191b0,
2780                 0x191d0, 0x191e8,
2781                 0x19238, 0x1924c,
2782                 0x193f8, 0x19474,
2783                 0x19490, 0x194f8,
2784                 0x19800, 0x19f30,
2785                 0x1a000, 0x1a06c,
2786                 0x1a0b0, 0x1a120,
2787                 0x1a128, 0x1a138,
2788                 0x1a190, 0x1a1c4,
2789                 0x1a1fc, 0x1a1fc,
2790                 0x1e040, 0x1e04c,
2791                 0x1e284, 0x1e28c,
2792                 0x1e2c0, 0x1e2c0,
2793                 0x1e2e0, 0x1e2e0,
2794                 0x1e300, 0x1e384,
2795                 0x1e3c0, 0x1e3c8,
2796                 0x1e440, 0x1e44c,
2797                 0x1e684, 0x1e68c,
2798                 0x1e6c0, 0x1e6c0,
2799                 0x1e6e0, 0x1e6e0,
2800                 0x1e700, 0x1e784,
2801                 0x1e7c0, 0x1e7c8,
2802                 0x1e840, 0x1e84c,
2803                 0x1ea84, 0x1ea8c,
2804                 0x1eac0, 0x1eac0,
2805                 0x1eae0, 0x1eae0,
2806                 0x1eb00, 0x1eb84,
2807                 0x1ebc0, 0x1ebc8,
2808                 0x1ec40, 0x1ec4c,
2809                 0x1ee84, 0x1ee8c,
2810                 0x1eec0, 0x1eec0,
2811                 0x1eee0, 0x1eee0,
2812                 0x1ef00, 0x1ef84,
2813                 0x1efc0, 0x1efc8,
2814                 0x1f040, 0x1f04c,
2815                 0x1f284, 0x1f28c,
2816                 0x1f2c0, 0x1f2c0,
2817                 0x1f2e0, 0x1f2e0,
2818                 0x1f300, 0x1f384,
2819                 0x1f3c0, 0x1f3c8,
2820                 0x1f440, 0x1f44c,
2821                 0x1f684, 0x1f68c,
2822                 0x1f6c0, 0x1f6c0,
2823                 0x1f6e0, 0x1f6e0,
2824                 0x1f700, 0x1f784,
2825                 0x1f7c0, 0x1f7c8,
2826                 0x1f840, 0x1f84c,
2827                 0x1fa84, 0x1fa8c,
2828                 0x1fac0, 0x1fac0,
2829                 0x1fae0, 0x1fae0,
2830                 0x1fb00, 0x1fb84,
2831                 0x1fbc0, 0x1fbc8,
2832                 0x1fc40, 0x1fc4c,
2833                 0x1fe84, 0x1fe8c,
2834                 0x1fec0, 0x1fec0,
2835                 0x1fee0, 0x1fee0,
2836                 0x1ff00, 0x1ff84,
2837                 0x1ffc0, 0x1ffc8,
2838                 0x20000, 0x2002c,
2839                 0x20100, 0x2013c,
2840                 0x20190, 0x201c8,
2841                 0x20200, 0x20318,
2842                 0x20400, 0x20528,
2843                 0x20540, 0x20614,
2844                 0x21000, 0x21040,
2845                 0x2104c, 0x21060,
2846                 0x210c0, 0x210ec,
2847                 0x21200, 0x21268,
2848                 0x21270, 0x21284,
2849                 0x212fc, 0x21388,
2850                 0x21400, 0x21404,
2851                 0x21500, 0x21518,
2852                 0x2152c, 0x2153c,
2853                 0x21550, 0x21554,
2854                 0x21600, 0x21600,
2855                 0x21608, 0x21628,
2856                 0x21630, 0x2163c,
2857                 0x21700, 0x2171c,
2858                 0x21780, 0x2178c,
2859                 0x21800, 0x21c38,
2860                 0x21c80, 0x21d7c,
2861                 0x21e00, 0x21e04,
2862                 0x22000, 0x2202c,
2863                 0x22100, 0x2213c,
2864                 0x22190, 0x221c8,
2865                 0x22200, 0x22318,
2866                 0x22400, 0x22528,
2867                 0x22540, 0x22614,
2868                 0x23000, 0x23040,
2869                 0x2304c, 0x23060,
2870                 0x230c0, 0x230ec,
2871                 0x23200, 0x23268,
2872                 0x23270, 0x23284,
2873                 0x232fc, 0x23388,
2874                 0x23400, 0x23404,
2875                 0x23500, 0x23518,
2876                 0x2352c, 0x2353c,
2877                 0x23550, 0x23554,
2878                 0x23600, 0x23600,
2879                 0x23608, 0x23628,
2880                 0x23630, 0x2363c,
2881                 0x23700, 0x2371c,
2882                 0x23780, 0x2378c,
2883                 0x23800, 0x23c38,
2884                 0x23c80, 0x23d7c,
2885                 0x23e00, 0x23e04,
2886                 0x24000, 0x2402c,
2887                 0x24100, 0x2413c,
2888                 0x24190, 0x241c8,
2889                 0x24200, 0x24318,
2890                 0x24400, 0x24528,
2891                 0x24540, 0x24614,
2892                 0x25000, 0x25040,
2893                 0x2504c, 0x25060,
2894                 0x250c0, 0x250ec,
2895                 0x25200, 0x25268,
2896                 0x25270, 0x25284,
2897                 0x252fc, 0x25388,
2898                 0x25400, 0x25404,
2899                 0x25500, 0x25518,
2900                 0x2552c, 0x2553c,
2901                 0x25550, 0x25554,
2902                 0x25600, 0x25600,
2903                 0x25608, 0x25628,
2904                 0x25630, 0x2563c,
2905                 0x25700, 0x2571c,
2906                 0x25780, 0x2578c,
2907                 0x25800, 0x25c38,
2908                 0x25c80, 0x25d7c,
2909                 0x25e00, 0x25e04,
2910                 0x26000, 0x2602c,
2911                 0x26100, 0x2613c,
2912                 0x26190, 0x261c8,
2913                 0x26200, 0x26318,
2914                 0x26400, 0x26528,
2915                 0x26540, 0x26614,
2916                 0x27000, 0x27040,
2917                 0x2704c, 0x27060,
2918                 0x270c0, 0x270ec,
2919                 0x27200, 0x27268,
2920                 0x27270, 0x27284,
2921                 0x272fc, 0x27388,
2922                 0x27400, 0x27404,
2923                 0x27500, 0x27518,
2924                 0x2752c, 0x2753c,
2925                 0x27550, 0x27554,
2926                 0x27600, 0x27600,
2927                 0x27608, 0x27628,
2928                 0x27630, 0x2763c,
2929                 0x27700, 0x2771c,
2930                 0x27780, 0x2778c,
2931                 0x27800, 0x27c38,
2932                 0x27c80, 0x27d7c,
2933                 0x27e00, 0x27e04
2934         };
2935
2936         regs->version = 4 | (sc->params.rev << 10);
2937         for (i = 0; i < nitems(reg_ranges); i += 2)
2938                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
2939 }
2940
2941 static void
2942 cxgbe_tick(void *arg)
2943 {
2944         struct port_info *pi = arg;
2945         struct ifnet *ifp = pi->ifp;
2946         struct sge_txq *txq;
2947         int i, drops;
2948         struct port_stats *s = &pi->stats;
2949
2950         PORT_LOCK(pi);
2951         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2952                 PORT_UNLOCK(pi);
2953                 return; /* without scheduling another callout */
2954         }
2955
2956         t4_get_port_stats(pi->adapter, pi->tx_chan, s);
2957
2958         ifp->if_opackets = s->tx_frames - s->tx_pause;
2959         ifp->if_ipackets = s->rx_frames - s->rx_pause;
2960         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
2961         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
2962         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
2963         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
2964         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2965             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
2966             s->rx_trunc3;
2967
2968         drops = s->tx_drop;
2969         for_each_txq(pi, i, txq)
2970                 drops += txq->br->br_drops;
2971         ifp->if_snd.ifq_drops = drops;
2972
2973         ifp->if_oerrors = s->tx_error_frames;
2974         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
2975             s->rx_fcs_err + s->rx_len_err;
2976
2977         callout_schedule(&pi->tick, hz);
2978         PORT_UNLOCK(pi);
2979 }
2980
2981 static void
2982 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
2983 {
2984         struct ifnet *vlan;
2985
2986         if (arg != ifp || ifp->if_type != IFT_ETHER)
2987                 return;
2988
2989         vlan = VLAN_DEVAT(ifp, vid);
2990         VLAN_SETCOOKIE(vlan, ifp);
2991 }
2992
2993 static int
2994 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
2995 {
2996
2997 #ifdef INVARIANTS
2998         panic("%s: opcode 0x%02x on iq %p with payload %p",
2999             __func__, rss->opcode, iq, m);
3000 #else
3001         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
3002             __func__, rss->opcode, iq, m);
3003         m_freem(m);
3004 #endif
3005         return (EDOOFUS);
3006 }
3007
3008 int
3009 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3010 {
3011         uintptr_t *loc, new;
3012
3013         if (opcode >= nitems(sc->cpl_handler))
3014                 return (EINVAL);
3015
3016         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3017         loc = (uintptr_t *) &sc->cpl_handler[opcode];
3018         atomic_store_rel_ptr(loc, new);
3019
3020         return (0);
3021 }
3022
3023 static int
3024 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
3025 {
3026
3027 #ifdef INVARIANTS
3028         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
3029 #else
3030         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
3031             __func__, iq, ctrl);
3032 #endif
3033         return (EDOOFUS);
3034 }
3035
3036 int
3037 t4_register_an_handler(struct adapter *sc, an_handler_t h)
3038 {
3039         uintptr_t *loc, new;
3040
3041         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
3042         loc = (uintptr_t *) &sc->an_handler;
3043         atomic_store_rel_ptr(loc, new);
3044
3045         return (0);
3046 }
3047
3048 static int
3049 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
3050 {
3051         const struct cpl_fw6_msg *cpl =
3052             __containerof(rpl, struct cpl_fw6_msg, data[0]);
3053
3054 #ifdef INVARIANTS
3055         panic("%s: fw_msg type %d", __func__, cpl->type);
3056 #else
3057         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
3058 #endif
3059         return (EDOOFUS);
3060 }
3061
3062 int
3063 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
3064 {
3065         uintptr_t *loc, new;
3066
3067         if (type >= nitems(sc->fw_msg_handler))
3068                 return (EINVAL);
3069
3070         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
3071         loc = (uintptr_t *) &sc->fw_msg_handler[type];
3072         atomic_store_rel_ptr(loc, new);
3073
3074         return (0);
3075 }
3076
3077 static int
3078 t4_sysctls(struct adapter *sc)
3079 {
3080         struct sysctl_ctx_list *ctx;
3081         struct sysctl_oid *oid;
3082         struct sysctl_oid_list *children, *c0;
3083         static char *caps[] = {
3084                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
3085                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",       /* caps[1] niccaps */
3086                 "\20\1TOE",                             /* caps[2] toecaps */
3087                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
3088                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
3089                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
3090                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
3091                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
3092         };
3093
3094         ctx = device_get_sysctl_ctx(sc->dev);
3095
3096         /*
3097          * dev.t4nex.X.
3098          */
3099         oid = device_get_sysctl_tree(sc->dev);
3100         c0 = children = SYSCTL_CHILDREN(oid);
3101
3102         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3103             &sc->params.nports, 0, "# of ports");
3104
3105         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3106             &sc->params.rev, 0, "chip hardware revision");
3107
3108         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3109             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3110
3111         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3112             CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
3113
3114         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3115             &sc->cfcsum, 0, "config file checksum");
3116
3117         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3118             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3119             sysctl_bitfield, "A", "available link capabilities");
3120
3121         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3122             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3123             sysctl_bitfield, "A", "available NIC capabilities");
3124
3125         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3126             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3127             sysctl_bitfield, "A", "available TCP offload capabilities");
3128
3129         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3130             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3131             sysctl_bitfield, "A", "available RDMA capabilities");
3132
3133         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3134             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3135             sysctl_bitfield, "A", "available iSCSI capabilities");
3136
3137         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3138             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3139             sysctl_bitfield, "A", "available FCoE capabilities");
3140
3141         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3142             &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3143
3144         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3145             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3146             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3147             "interrupt holdoff timer values (us)");
3148
3149         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3150             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3151             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3152             "interrupt holdoff packet counter values");
3153
3154 #ifdef SBUF_DRAIN
3155         /*
3156          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3157          */
3158         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3159             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3160             "logs and miscellaneous information");
3161         children = SYSCTL_CHILDREN(oid);
3162
3163         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3164             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3165             sysctl_cctrl, "A", "congestion control");
3166
3167         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3168             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3169             sysctl_cpl_stats, "A", "CPL statistics");
3170
3171         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3172             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3173             sysctl_ddp_stats, "A", "DDP statistics");
3174
3175         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3176             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3177             sysctl_devlog, "A", "firmware's device log");
3178
3179         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3180             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3181             sysctl_fcoe_stats, "A", "FCoE statistics");
3182
3183         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3184             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3185             sysctl_hw_sched, "A", "hardware scheduler ");
3186
3187         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3188             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3189             sysctl_l2t, "A", "hardware L2 table");
3190
3191         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3192             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3193             sysctl_lb_stats, "A", "loopback statistics");
3194
3195         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3196             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3197             sysctl_meminfo, "A", "memory regions");
3198
3199         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3200             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3201             sysctl_path_mtus, "A", "path MTUs");
3202
3203         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3204             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3205             sysctl_pm_stats, "A", "PM statistics");
3206
3207         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3208             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3209             sysctl_rdma_stats, "A", "RDMA statistics");
3210
3211         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3212             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3213             sysctl_tcp_stats, "A", "TCP statistics");
3214
3215         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3216             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3217             sysctl_tids, "A", "TID information");
3218
3219         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3220             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3221             sysctl_tp_err_stats, "A", "TP error statistics");
3222
3223         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3224             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3225             sysctl_tx_rate, "A", "Tx rate");
3226 #endif
3227
3228 #ifdef TCP_OFFLOAD
3229         if (is_offload(sc)) {
3230                 /*
3231                  * dev.t4nex.X.toe.
3232                  */
3233                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3234                     NULL, "TOE parameters");
3235                 children = SYSCTL_CHILDREN(oid);
3236
3237                 sc->tt.sndbuf = 256 * 1024;
3238                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3239                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
3240
3241                 sc->tt.ddp = 0;
3242                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3243                     &sc->tt.ddp, 0, "DDP allowed");
3244
3245                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
3246                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3247                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
3248
3249                 sc->tt.ddp_thres =
3250                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
3251                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3252                     &sc->tt.ddp_thres, 0, "DDP threshold");
3253         }
3254 #endif
3255
3256
3257         return (0);
3258 }
3259
3260 static int
3261 cxgbe_sysctls(struct port_info *pi)
3262 {
3263         struct sysctl_ctx_list *ctx;
3264         struct sysctl_oid *oid;
3265         struct sysctl_oid_list *children;
3266
3267         ctx = device_get_sysctl_ctx(pi->dev);
3268
3269         /*
3270          * dev.cxgbe.X.
3271          */
3272         oid = device_get_sysctl_tree(pi->dev);
3273         children = SYSCTL_CHILDREN(oid);
3274
3275         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3276             &pi->nrxq, 0, "# of rx queues");
3277         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3278             &pi->ntxq, 0, "# of tx queues");
3279         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3280             &pi->first_rxq, 0, "index of first rx queue");
3281         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3282             &pi->first_txq, 0, "index of first tx queue");
3283
3284 #ifdef TCP_OFFLOAD
3285         if (is_offload(pi->adapter)) {
3286                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3287                     &pi->nofldrxq, 0,
3288                     "# of rx queues for offloaded TCP connections");
3289                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3290                     &pi->nofldtxq, 0,
3291                     "# of tx queues for offloaded TCP connections");
3292                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3293                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3294                     "index of first TOE rx queue");
3295                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3296                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
3297                     "index of first TOE tx queue");
3298         }
3299 #endif
3300
3301         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3302             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3303             "holdoff timer index");
3304         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3305             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3306             "holdoff packet counter index");
3307
3308         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3309             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3310             "rx queue size");
3311         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3312             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3313             "tx queue size");
3314
3315         /*
3316          * dev.cxgbe.X.stats.
3317          */
3318         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3319             NULL, "port statistics");
3320         children = SYSCTL_CHILDREN(oid);
3321
3322 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3323         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3324             CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3325             sysctl_handle_t4_reg64, "QU", desc)
3326
3327         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3328             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3329         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3330             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3331         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3332             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3333         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3334             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3335         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3336             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3337         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3338             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3339         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3340             "# of tx frames in this range",
3341             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3342         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3343             "# of tx frames in this range",
3344             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3345         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3346             "# of tx frames in this range",
3347             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3348         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3349             "# of tx frames in this range",
3350             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3351         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3352             "# of tx frames in this range",
3353             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3354         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3355             "# of tx frames in this range",
3356             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3357         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3358             "# of tx frames in this range",
3359             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3360         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3361             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3362         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3363             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3364         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3365             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3366         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3367             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3368         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3369             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3370         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3371             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3372         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3373             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3374         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3375             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3376         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3377             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3378         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3379             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3380
3381         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3382             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3383         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3384             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3385         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3386             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3387         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3388             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3389         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3390             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3391         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3392             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3393         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3394             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3395         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3396             "# of frames received with bad FCS",
3397             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3398         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3399             "# of frames received with length error",
3400             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3401         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3402             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3403         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3404             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3405         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3406             "# of rx frames in this range",
3407             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3408         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3409             "# of rx frames in this range",
3410             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3411         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3412             "# of rx frames in this range",
3413             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3414         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3415             "# of rx frames in this range",
3416             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3417         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3418             "# of rx frames in this range",
3419             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3420         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3421             "# of rx frames in this range",
3422             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3423         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3424             "# of rx frames in this range",
3425             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3426         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3427             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3428         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3429             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3430         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3431             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3432         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3433             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3434         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3435             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3436         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3437             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3438         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3439             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3440         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3441             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3442         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3443             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3444
3445 #undef SYSCTL_ADD_T4_REG64
3446
3447 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3448         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3449             &pi->stats.name, desc)
3450
3451         /* We get these from port_stats and they may be stale by upto 1s */
3452         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3453             "# drops due to buffer-group 0 overflows");
3454         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3455             "# drops due to buffer-group 1 overflows");
3456         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3457             "# drops due to buffer-group 2 overflows");
3458         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3459             "# drops due to buffer-group 3 overflows");
3460         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3461             "# of buffer-group 0 truncated packets");
3462         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3463             "# of buffer-group 1 truncated packets");
3464         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3465             "# of buffer-group 2 truncated packets");
3466         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3467             "# of buffer-group 3 truncated packets");
3468
3469 #undef SYSCTL_ADD_T4_PORTSTAT
3470
3471         return (0);
3472 }
3473
3474 static int
3475 sysctl_int_array(SYSCTL_HANDLER_ARGS)
3476 {
3477         int rc, *i;
3478         struct sbuf sb;
3479
3480         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3481         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3482                 sbuf_printf(&sb, "%d ", *i);
3483         sbuf_trim(&sb);
3484         sbuf_finish(&sb);
3485         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3486         sbuf_delete(&sb);
3487         return (rc);
3488 }
3489
3490 static int
3491 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
3492 {
3493         int rc;
3494         struct sbuf *sb;
3495
3496         rc = sysctl_wire_old_buffer(req, 0);
3497         if (rc != 0)
3498                 return(rc);
3499
3500         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3501         if (sb == NULL)
3502                 return (ENOMEM);
3503
3504         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3505         rc = sbuf_finish(sb);
3506         sbuf_delete(sb);
3507
3508         return (rc);
3509 }
3510
3511 static int
3512 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3513 {
3514         struct port_info *pi = arg1;
3515         struct adapter *sc = pi->adapter;
3516         int idx, rc, i;
3517         struct sge_rxq *rxq;
3518         uint8_t v;
3519
3520         idx = pi->tmr_idx;
3521
3522         rc = sysctl_handle_int(oidp, &idx, 0, req);
3523         if (rc != 0 || req->newptr == NULL)
3524                 return (rc);
3525
3526         if (idx < 0 || idx >= SGE_NTIMERS)
3527                 return (EINVAL);
3528
3529         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3530             "t4tmr");
3531         if (rc)
3532                 return (rc);
3533
3534         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3535         for_each_rxq(pi, i, rxq) {
3536 #ifdef atomic_store_rel_8
3537                 atomic_store_rel_8(&rxq->iq.intr_params, v);
3538 #else
3539                 rxq->iq.intr_params = v;
3540 #endif
3541         }
3542         pi->tmr_idx = idx;
3543
3544         end_synchronized_op(sc, LOCK_HELD);
3545         return (0);
3546 }
3547
3548 static int
3549 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3550 {
3551         struct port_info *pi = arg1;
3552         struct adapter *sc = pi->adapter;
3553         int idx, rc;
3554
3555         idx = pi->pktc_idx;
3556
3557         rc = sysctl_handle_int(oidp, &idx, 0, req);
3558         if (rc != 0 || req->newptr == NULL)
3559                 return (rc);
3560
3561         if (idx < -1 || idx >= SGE_NCOUNTERS)
3562                 return (EINVAL);
3563
3564         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3565             "t4pktc");
3566         if (rc)
3567                 return (rc);
3568
3569         if (pi->flags & PORT_INIT_DONE)
3570                 rc = EBUSY; /* cannot be changed once the queues are created */
3571         else
3572                 pi->pktc_idx = idx;
3573
3574         end_synchronized_op(sc, LOCK_HELD);
3575         return (rc);
3576 }
3577
3578 static int
3579 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3580 {
3581         struct port_info *pi = arg1;
3582         struct adapter *sc = pi->adapter;
3583         int qsize, rc;
3584
3585         qsize = pi->qsize_rxq;
3586
3587         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3588         if (rc != 0 || req->newptr == NULL)
3589                 return (rc);
3590
3591         if (qsize < 128 || (qsize & 7))
3592                 return (EINVAL);
3593
3594         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3595             "t4rxqs");
3596         if (rc)
3597                 return (rc);
3598
3599         if (pi->flags & PORT_INIT_DONE)
3600                 rc = EBUSY; /* cannot be changed once the queues are created */
3601         else
3602                 pi->qsize_rxq = qsize;
3603
3604         end_synchronized_op(sc, LOCK_HELD);
3605         return (rc);
3606 }
3607
3608 static int
3609 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3610 {
3611         struct port_info *pi = arg1;
3612         struct adapter *sc = pi->adapter;
3613         int qsize, rc;
3614
3615         qsize = pi->qsize_txq;
3616
3617         rc = sysctl_handle_int(oidp, &qsize, 0, req);
3618         if (rc != 0 || req->newptr == NULL)
3619                 return (rc);
3620
3621         /* bufring size must be powerof2 */
3622         if (qsize < 128 || !powerof2(qsize))
3623                 return (EINVAL);
3624
3625         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3626             "t4txqs");
3627         if (rc)
3628                 return (rc);
3629
3630         if (pi->flags & PORT_INIT_DONE)
3631                 rc = EBUSY; /* cannot be changed once the queues are created */
3632         else
3633                 pi->qsize_txq = qsize;
3634
3635         end_synchronized_op(sc, LOCK_HELD);
3636         return (rc);
3637 }
3638
3639 static int
3640 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3641 {
3642         struct adapter *sc = arg1;
3643         int reg = arg2;
3644         uint64_t val;
3645
3646         val = t4_read_reg64(sc, reg);
3647
3648         return (sysctl_handle_64(oidp, &val, 0, req));
3649 }
3650
3651 #ifdef SBUF_DRAIN
3652 static int
3653 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
3654 {
3655         struct adapter *sc = arg1;
3656         struct sbuf *sb;
3657         int rc, i;
3658         uint16_t incr[NMTUS][NCCTRL_WIN];
3659         static const char *dec_fac[] = {
3660                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3661                 "0.9375"
3662         };
3663
3664         rc = sysctl_wire_old_buffer(req, 0);
3665         if (rc != 0)
3666                 return (rc);
3667
3668         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3669         if (sb == NULL)
3670                 return (ENOMEM);
3671
3672         t4_read_cong_tbl(sc, incr);
3673
3674         for (i = 0; i < NCCTRL_WIN; ++i) {
3675                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3676                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3677                     incr[5][i], incr[6][i], incr[7][i]);
3678                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3679                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3680                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3681                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3682         }
3683
3684         rc = sbuf_finish(sb);
3685         sbuf_delete(sb);
3686
3687         return (rc);
3688 }
3689
3690 static int
3691 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
3692 {
3693         struct adapter *sc = arg1;
3694         struct sbuf *sb;
3695         int rc;
3696         struct tp_cpl_stats stats;
3697
3698         rc = sysctl_wire_old_buffer(req, 0);
3699         if (rc != 0)
3700                 return (rc);
3701
3702         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3703         if (sb == NULL)
3704                 return (ENOMEM);
3705
3706         t4_tp_get_cpl_stats(sc, &stats);
3707
3708         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
3709             "channel 3\n");
3710         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
3711                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
3712         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
3713                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
3714
3715         rc = sbuf_finish(sb);
3716         sbuf_delete(sb);
3717
3718         return (rc);
3719 }
3720
3721 static int
3722 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
3723 {
3724         struct adapter *sc = arg1;
3725         struct sbuf *sb;
3726         int rc;
3727         struct tp_usm_stats stats;
3728
3729         rc = sysctl_wire_old_buffer(req, 0);
3730         if (rc != 0)
3731                 return(rc);
3732
3733         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3734         if (sb == NULL)
3735                 return (ENOMEM);
3736
3737         t4_get_usm_stats(sc, &stats);
3738
3739         sbuf_printf(sb, "Frames: %u\n", stats.frames);
3740         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
3741         sbuf_printf(sb, "Drops:  %u", stats.drops);
3742
3743         rc = sbuf_finish(sb);
3744         sbuf_delete(sb);
3745
3746         return (rc);
3747 }
3748
3749 const char *devlog_level_strings[] = {
3750         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
3751         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
3752         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
3753         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
3754         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
3755         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
3756 };
3757
3758 const char *devlog_facility_strings[] = {
3759         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
3760         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
3761         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
3762         [FW_DEVLOG_FACILITY_RES]        = "RES",
3763         [FW_DEVLOG_FACILITY_HW]         = "HW",
3764         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
3765         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
3766         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
3767         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
3768         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
3769         [FW_DEVLOG_FACILITY_VI]         = "VI",
3770         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
3771         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
3772         [FW_DEVLOG_FACILITY_TM]         = "TM",
3773         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
3774         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
3775         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
3776         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
3777         [FW_DEVLOG_FACILITY_RI]         = "RI",
3778         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
3779         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
3780         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
3781         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
3782 };
3783
3784 static int
3785 sysctl_devlog(SYSCTL_HANDLER_ARGS)
3786 {
3787         struct adapter *sc = arg1;
3788         struct devlog_params *dparams = &sc->params.devlog;
3789         struct fw_devlog_e *buf, *e;
3790         int i, j, rc, nentries, first = 0;
3791         struct sbuf *sb;
3792         uint64_t ftstamp = UINT64_MAX;
3793
3794         if (dparams->start == 0)
3795                 return (ENXIO);
3796
3797         nentries = dparams->size / sizeof(struct fw_devlog_e);
3798
3799         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
3800         if (buf == NULL)
3801                 return (ENOMEM);
3802
3803         rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
3804             (void *)buf);
3805         if (rc != 0)
3806                 goto done;
3807
3808         for (i = 0; i < nentries; i++) {
3809                 e = &buf[i];
3810
3811                 if (e->timestamp == 0)
3812                         break;  /* end */
3813
3814                 e->timestamp = be64toh(e->timestamp);
3815                 e->seqno = be32toh(e->seqno);
3816                 for (j = 0; j < 8; j++)
3817                         e->params[j] = be32toh(e->params[j]);
3818
3819                 if (e->timestamp < ftstamp) {
3820                         ftstamp = e->timestamp;
3821                         first = i;
3822                 }
3823         }
3824
3825         if (buf[first].timestamp == 0)
3826                 goto done;      /* nothing in the log */
3827
3828         rc = sysctl_wire_old_buffer(req, 0);
3829         if (rc != 0)
3830                 goto done;
3831
3832         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3833         if (sb == NULL) {
3834                 rc = ENOMEM;
3835                 goto done;
3836         }
3837         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
3838             "Seq#", "Tstamp", "Level", "Facility", "Message");
3839
3840         i = first;
3841         do {
3842                 e = &buf[i];
3843                 if (e->timestamp == 0)
3844                         break;  /* end */
3845
3846                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
3847                     e->seqno, e->timestamp,
3848                     (e->level < nitems(devlog_level_strings) ?
3849                         devlog_level_strings[e->level] : "UNKNOWN"),
3850                     (e->facility < nitems(devlog_facility_strings) ?
3851                         devlog_facility_strings[e->facility] : "UNKNOWN"));
3852                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
3853                     e->params[2], e->params[3], e->params[4],
3854                     e->params[5], e->params[6], e->params[7]);
3855
3856                 if (++i == nentries)
3857                         i = 0;
3858         } while (i != first);
3859
3860         rc = sbuf_finish(sb);
3861         sbuf_delete(sb);
3862 done:
3863         free(buf, M_CXGBE);
3864         return (rc);
3865 }
3866
3867 static int
3868 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
3869 {
3870         struct adapter *sc = arg1;
3871         struct sbuf *sb;
3872         int rc;
3873         struct tp_fcoe_stats stats[4];
3874
3875         rc = sysctl_wire_old_buffer(req, 0);
3876         if (rc != 0)
3877                 return (rc);
3878
3879         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3880         if (sb == NULL)
3881                 return (ENOMEM);
3882
3883         t4_get_fcoe_stats(sc, 0, &stats[0]);
3884         t4_get_fcoe_stats(sc, 1, &stats[1]);
3885         t4_get_fcoe_stats(sc, 2, &stats[2]);
3886         t4_get_fcoe_stats(sc, 3, &stats[3]);
3887
3888         sbuf_printf(sb, "                   channel 0        channel 1        "
3889             "channel 2        channel 3\n");
3890         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
3891             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
3892             stats[3].octetsDDP);
3893         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
3894             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
3895         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
3896             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
3897             stats[3].framesDrop);
3898
3899         rc = sbuf_finish(sb);
3900         sbuf_delete(sb);
3901
3902         return (rc);
3903 }
3904
3905 static int
3906 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
3907 {
3908         struct adapter *sc = arg1;
3909         struct sbuf *sb;
3910         int rc, i;
3911         unsigned int map, kbps, ipg, mode;
3912         unsigned int pace_tab[NTX_SCHED];
3913
3914         rc = sysctl_wire_old_buffer(req, 0);
3915         if (rc != 0)
3916                 return (rc);
3917
3918         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3919         if (sb == NULL)
3920                 return (ENOMEM);
3921
3922         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
3923         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
3924         t4_read_pace_tbl(sc, pace_tab);
3925
3926         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
3927             "Class IPG (0.1 ns)   Flow IPG (us)");
3928
3929         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
3930                 t4_get_tx_sched(sc, i, &kbps, &ipg);
3931                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
3932                     (mode & (1 << i)) ? "flow" : "class", map & 3);
3933                 if (kbps)
3934                         sbuf_printf(sb, "%9u     ", kbps);
3935                 else
3936                         sbuf_printf(sb, " disabled     ");
3937
3938                 if (ipg)
3939                         sbuf_printf(sb, "%13u        ", ipg);
3940                 else
3941                         sbuf_printf(sb, "     disabled        ");
3942
3943                 if (pace_tab[i])
3944                         sbuf_printf(sb, "%10u", pace_tab[i]);
3945                 else
3946                         sbuf_printf(sb, "  disabled");
3947         }
3948
3949         rc = sbuf_finish(sb);
3950         sbuf_delete(sb);
3951
3952         return (rc);
3953 }
3954
3955 static int
3956 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
3957 {
3958         struct adapter *sc = arg1;
3959         struct sbuf *sb;
3960         int rc, i, j;
3961         uint64_t *p0, *p1;
3962         struct lb_port_stats s[2];
3963         static const char *stat_name[] = {
3964                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
3965                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
3966                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
3967                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
3968                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
3969                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
3970                 "BG2FramesTrunc:", "BG3FramesTrunc:"
3971         };
3972
3973         rc = sysctl_wire_old_buffer(req, 0);
3974         if (rc != 0)
3975                 return (rc);
3976
3977         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3978         if (sb == NULL)
3979                 return (ENOMEM);
3980
3981         memset(s, 0, sizeof(s));
3982
3983         for (i = 0; i < 4; i += 2) {
3984                 t4_get_lb_stats(sc, i, &s[0]);
3985                 t4_get_lb_stats(sc, i + 1, &s[1]);
3986
3987                 p0 = &s[0].octets;
3988                 p1 = &s[1].octets;
3989                 sbuf_printf(sb, "%s                       Loopback %u"
3990                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
3991
3992                 for (j = 0; j < nitems(stat_name); j++)
3993                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
3994                                    *p0++, *p1++);
3995         }
3996
3997         rc = sbuf_finish(sb);
3998         sbuf_delete(sb);
3999
4000         return (rc);
4001 }
4002
4003 struct mem_desc {
4004         unsigned int base;
4005         unsigned int limit;
4006         unsigned int idx;
4007 };
4008
4009 static int
4010 mem_desc_cmp(const void *a, const void *b)
4011 {
4012         return ((const struct mem_desc *)a)->base -
4013                ((const struct mem_desc *)b)->base;
4014 }
4015
4016 static void
4017 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
4018     unsigned int to)
4019 {
4020         unsigned int size;
4021
4022         size = to - from + 1;
4023         if (size == 0)
4024                 return;
4025
4026         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
4027         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
4028 }
4029
4030 static int
4031 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
4032 {
4033         struct adapter *sc = arg1;
4034         struct sbuf *sb;
4035         int rc, i, n;
4036         uint32_t lo, hi;
4037         static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
4038         static const char *region[] = {
4039                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
4040                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
4041                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
4042                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
4043                 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
4044                 "ULPTX state:", "On-chip queues:"
4045         };
4046         struct mem_desc avail[3];
4047         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
4048         struct mem_desc *md = mem;
4049
4050         rc = sysctl_wire_old_buffer(req, 0);
4051         if (rc != 0)
4052                 return (rc);
4053
4054         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4055         if (sb == NULL)
4056                 return (ENOMEM);
4057
4058         for (i = 0; i < nitems(mem); i++) {
4059                 mem[i].limit = 0;
4060                 mem[i].idx = i;
4061         }
4062
4063         /* Find and sort the populated memory ranges */
4064         i = 0;
4065         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4066         if (lo & F_EDRAM0_ENABLE) {
4067                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4068                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
4069                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
4070                 avail[i].idx = 0;
4071                 i++;
4072         }
4073         if (lo & F_EDRAM1_ENABLE) {
4074                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4075                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
4076                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
4077                 avail[i].idx = 1;
4078                 i++;
4079         }
4080         if (lo & F_EXT_MEM_ENABLE) {
4081                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4082                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
4083                 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
4084                 avail[i].idx = 2;
4085                 i++;
4086         }
4087         if (!i)                                    /* no memory available */
4088                 return 0;
4089         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
4090
4091         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
4092         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
4093         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
4094         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4095         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
4096         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
4097         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
4098         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
4099         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
4100
4101         /* the next few have explicit upper bounds */
4102         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
4103         md->limit = md->base - 1 +
4104                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
4105                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
4106         md++;
4107
4108         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4109         md->limit = md->base - 1 +
4110                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4111                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4112         md++;
4113
4114         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4115                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4116                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4117                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4118         } else {
4119                 md->base = 0;
4120                 md->idx = nitems(region);  /* hide it */
4121         }
4122         md++;
4123
4124 #define ulp_region(reg) \
4125         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4126         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4127
4128         ulp_region(RX_ISCSI);
4129         ulp_region(RX_TDDP);
4130         ulp_region(TX_TPT);
4131         ulp_region(RX_STAG);
4132         ulp_region(RX_RQ);
4133         ulp_region(RX_RQUDP);
4134         ulp_region(RX_PBL);
4135         ulp_region(TX_PBL);
4136 #undef ulp_region
4137
4138         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4139         md->limit = md->base + sc->tids.ntids - 1;
4140         md++;
4141         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4142         md->limit = md->base + sc->tids.ntids - 1;
4143         md++;
4144
4145         md->base = sc->vres.ocq.start;
4146         if (sc->vres.ocq.size)
4147                 md->limit = md->base + sc->vres.ocq.size - 1;
4148         else
4149                 md->idx = nitems(region);  /* hide it */
4150         md++;
4151
4152         /* add any address-space holes, there can be up to 3 */
4153         for (n = 0; n < i - 1; n++)
4154                 if (avail[n].limit < avail[n + 1].base)
4155                         (md++)->base = avail[n].limit;
4156         if (avail[n].limit)
4157                 (md++)->base = avail[n].limit;
4158
4159         n = md - mem;
4160         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4161
4162         for (lo = 0; lo < i; lo++)
4163                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4164                                 avail[lo].limit - 1);
4165
4166         sbuf_printf(sb, "\n");
4167         for (i = 0; i < n; i++) {
4168                 if (mem[i].idx >= nitems(region))
4169                         continue;                        /* skip holes */
4170                 if (!mem[i].limit)
4171                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4172                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
4173                                 mem[i].limit);
4174         }
4175
4176         sbuf_printf(sb, "\n");
4177         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4178         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4179         mem_region_show(sb, "uP RAM:", lo, hi);
4180
4181         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4182         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4183         mem_region_show(sb, "uP Extmem2:", lo, hi);
4184
4185         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4186         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4187                    G_PMRXMAXPAGE(lo),
4188                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4189                    (lo & F_PMRXNUMCHN) ? 2 : 1);
4190
4191         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4192         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4193         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4194                    G_PMTXMAXPAGE(lo),
4195                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4196                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4197         sbuf_printf(sb, "%u p-structs\n",
4198                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4199
4200         for (i = 0; i < 4; i++) {
4201                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4202                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4203                            i, G_USED(lo), G_ALLOC(lo));
4204         }
4205         for (i = 0; i < 4; i++) {
4206                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4207                 sbuf_printf(sb,
4208                            "\nLoopback %d using %u pages out of %u allocated",
4209                            i, G_USED(lo), G_ALLOC(lo));
4210         }
4211
4212         rc = sbuf_finish(sb);
4213         sbuf_delete(sb);
4214
4215         return (rc);
4216 }
4217
4218 static int
4219 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4220 {
4221         struct adapter *sc = arg1;
4222         struct sbuf *sb;
4223         int rc;
4224         uint16_t mtus[NMTUS];
4225
4226         rc = sysctl_wire_old_buffer(req, 0);
4227         if (rc != 0)
4228                 return (rc);
4229
4230         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4231         if (sb == NULL)
4232                 return (ENOMEM);
4233
4234         t4_read_mtu_tbl(sc, mtus, NULL);
4235
4236         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4237             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4238             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4239             mtus[14], mtus[15]);
4240
4241         rc = sbuf_finish(sb);
4242         sbuf_delete(sb);
4243
4244         return (rc);
4245 }
4246
4247 static int
4248 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4249 {
4250         struct adapter *sc = arg1;
4251         struct sbuf *sb;
4252         int rc, i;
4253         uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4254         uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4255         static const char *pm_stats[] = {
4256                 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4257         };
4258
4259         rc = sysctl_wire_old_buffer(req, 0);
4260         if (rc != 0)
4261                 return (rc);
4262
4263         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4264         if (sb == NULL)
4265                 return (ENOMEM);
4266
4267         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4268         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4269
4270         sbuf_printf(sb, "                Tx count            Tx cycles    "
4271             "Rx count            Rx cycles");
4272         for (i = 0; i < PM_NSTATS; i++)
4273                 sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4274                     pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4275
4276         rc = sbuf_finish(sb);
4277         sbuf_delete(sb);
4278
4279         return (rc);
4280 }
4281
4282 static int
4283 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4284 {
4285         struct adapter *sc = arg1;
4286         struct sbuf *sb;
4287         int rc;
4288         struct tp_rdma_stats stats;
4289
4290         rc = sysctl_wire_old_buffer(req, 0);
4291         if (rc != 0)
4292                 return (rc);
4293
4294         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4295         if (sb == NULL)
4296                 return (ENOMEM);
4297
4298         t4_tp_get_rdma_stats(sc, &stats);
4299         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4300         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4301
4302         rc = sbuf_finish(sb);
4303         sbuf_delete(sb);
4304
4305         return (rc);
4306 }
4307
4308 static int
4309 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4310 {
4311         struct adapter *sc = arg1;
4312         struct sbuf *sb;
4313         int rc;
4314         struct tp_tcp_stats v4, v6;
4315
4316         rc = sysctl_wire_old_buffer(req, 0);
4317         if (rc != 0)
4318                 return (rc);
4319
4320         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4321         if (sb == NULL)
4322                 return (ENOMEM);
4323
4324         t4_tp_get_tcp_stats(sc, &v4, &v6);
4325         sbuf_printf(sb,
4326             "                                IP                 IPv6\n");
4327         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4328             v4.tcpOutRsts, v6.tcpOutRsts);
4329         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4330             v4.tcpInSegs, v6.tcpInSegs);
4331         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4332             v4.tcpOutSegs, v6.tcpOutSegs);
4333         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4334             v4.tcpRetransSegs, v6.tcpRetransSegs);
4335
4336         rc = sbuf_finish(sb);
4337         sbuf_delete(sb);
4338
4339         return (rc);
4340 }
4341
4342 static int
4343 sysctl_tids(SYSCTL_HANDLER_ARGS)
4344 {
4345         struct adapter *sc = arg1;
4346         struct sbuf *sb;
4347         int rc;
4348         struct tid_info *t = &sc->tids;
4349
4350         rc = sysctl_wire_old_buffer(req, 0);
4351         if (rc != 0)
4352                 return (rc);
4353
4354         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4355         if (sb == NULL)
4356                 return (ENOMEM);
4357
4358         if (t->natids) {
4359                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4360                     t->atids_in_use);
4361         }
4362
4363         if (t->ntids) {
4364                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4365                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4366
4367                         if (b) {
4368                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4369                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4370                                     t->ntids - 1);
4371                         } else {
4372                                 sbuf_printf(sb, "TID range: %u-%u",
4373                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4374                                     t->ntids - 1);
4375                         }
4376                 } else
4377                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4378                 sbuf_printf(sb, ", in use: %u\n",
4379                     atomic_load_acq_int(&t->tids_in_use));
4380         }
4381
4382         if (t->nstids) {
4383                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4384                     t->stid_base + t->nstids - 1, t->stids_in_use);
4385         }
4386
4387         if (t->nftids) {
4388                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4389                     t->ftid_base + t->nftids - 1);
4390         }
4391
4392         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4393             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4394             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4395
4396         rc = sbuf_finish(sb);
4397         sbuf_delete(sb);
4398
4399         return (rc);
4400 }
4401
4402 static int
4403 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4404 {
4405         struct adapter *sc = arg1;
4406         struct sbuf *sb;
4407         int rc;
4408         struct tp_err_stats stats;
4409
4410         rc = sysctl_wire_old_buffer(req, 0);
4411         if (rc != 0)
4412                 return (rc);
4413
4414         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4415         if (sb == NULL)
4416                 return (ENOMEM);
4417
4418         t4_tp_get_err_stats(sc, &stats);
4419
4420         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4421                       "channel 3\n");
4422         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4423             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4424             stats.macInErrs[3]);
4425         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4426             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4427             stats.hdrInErrs[3]);
4428         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4429             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4430             stats.tcpInErrs[3]);
4431         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4432             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4433             stats.tcp6InErrs[3]);
4434         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4435             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4436             stats.tnlCongDrops[3]);
4437         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4438             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4439             stats.tnlTxDrops[3]);
4440         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4441             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4442             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4443         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4444             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4445             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4446         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4447             stats.ofldNoNeigh, stats.ofldCongDefer);
4448
4449         rc = sbuf_finish(sb);
4450         sbuf_delete(sb);
4451
4452         return (rc);
4453 }
4454
4455 static int
4456 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4457 {
4458         struct adapter *sc = arg1;
4459         struct sbuf *sb;
4460         int rc;
4461         u64 nrate[NCHAN], orate[NCHAN];
4462
4463         rc = sysctl_wire_old_buffer(req, 0);
4464         if (rc != 0)
4465                 return (rc);
4466
4467         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4468         if (sb == NULL)
4469                 return (ENOMEM);
4470
4471         t4_get_chan_txrate(sc, nrate, orate);
4472         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4473                  "channel 3\n");
4474         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4475             nrate[0], nrate[1], nrate[2], nrate[3]);
4476         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4477             orate[0], orate[1], orate[2], orate[3]);
4478
4479         rc = sbuf_finish(sb);
4480         sbuf_delete(sb);
4481
4482         return (rc);
4483 }
4484 #endif
4485
4486 static inline void
4487 txq_start(struct ifnet *ifp, struct sge_txq *txq)
4488 {
4489         struct buf_ring *br;
4490         struct mbuf *m;
4491
4492         TXQ_LOCK_ASSERT_OWNED(txq);
4493
4494         br = txq->br;
4495         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4496         if (m)
4497                 t4_eth_tx(ifp, txq, m);
4498 }
4499
4500 void
4501 t4_tx_callout(void *arg)
4502 {
4503         struct sge_eq *eq = arg;
4504         struct adapter *sc;
4505
4506         if (EQ_TRYLOCK(eq) == 0)
4507                 goto reschedule;
4508
4509         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4510                 EQ_UNLOCK(eq);
4511 reschedule:
4512                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
4513                         callout_schedule(&eq->tx_callout, 1);
4514                 return;
4515         }
4516
4517         EQ_LOCK_ASSERT_OWNED(eq);
4518
4519         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4520
4521                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4522                         struct sge_txq *txq = arg;
4523                         struct port_info *pi = txq->ifp->if_softc;
4524
4525                         sc = pi->adapter;
4526                 } else {
4527                         struct sge_wrq *wrq = arg;
4528
4529                         sc = wrq->adapter;
4530                 }
4531
4532                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4533         }
4534
4535         EQ_UNLOCK(eq);
4536 }
4537
4538 void
4539 t4_tx_task(void *arg, int count)
4540 {
4541         struct sge_eq *eq = arg;
4542
4543         EQ_LOCK(eq);
4544         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4545                 struct sge_txq *txq = arg;
4546                 txq_start(txq->ifp, txq);
4547         } else {
4548                 struct sge_wrq *wrq = arg;
4549                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4550         }
4551         EQ_UNLOCK(eq);
4552 }
4553
4554 static uint32_t
4555 fconf_to_mode(uint32_t fconf)
4556 {
4557         uint32_t mode;
4558
4559         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4560             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4561
4562         if (fconf & F_FRAGMENTATION)
4563                 mode |= T4_FILTER_IP_FRAGMENT;
4564
4565         if (fconf & F_MPSHITTYPE)
4566                 mode |= T4_FILTER_MPS_HIT_TYPE;
4567
4568         if (fconf & F_MACMATCH)
4569                 mode |= T4_FILTER_MAC_IDX;
4570
4571         if (fconf & F_ETHERTYPE)
4572                 mode |= T4_FILTER_ETH_TYPE;
4573
4574         if (fconf & F_PROTOCOL)
4575                 mode |= T4_FILTER_IP_PROTO;
4576
4577         if (fconf & F_TOS)
4578                 mode |= T4_FILTER_IP_TOS;
4579
4580         if (fconf & F_VLAN)
4581                 mode |= T4_FILTER_VLAN;
4582
4583         if (fconf & F_VNIC_ID)
4584                 mode |= T4_FILTER_VNIC;
4585
4586         if (fconf & F_PORT)
4587                 mode |= T4_FILTER_PORT;
4588
4589         if (fconf & F_FCOE)
4590                 mode |= T4_FILTER_FCoE;
4591
4592         return (mode);
4593 }
4594
4595 static uint32_t
4596 mode_to_fconf(uint32_t mode)
4597 {
4598         uint32_t fconf = 0;
4599
4600         if (mode & T4_FILTER_IP_FRAGMENT)
4601                 fconf |= F_FRAGMENTATION;
4602
4603         if (mode & T4_FILTER_MPS_HIT_TYPE)
4604                 fconf |= F_MPSHITTYPE;
4605
4606         if (mode & T4_FILTER_MAC_IDX)
4607                 fconf |= F_MACMATCH;
4608
4609         if (mode & T4_FILTER_ETH_TYPE)
4610                 fconf |= F_ETHERTYPE;
4611
4612         if (mode & T4_FILTER_IP_PROTO)
4613                 fconf |= F_PROTOCOL;
4614
4615         if (mode & T4_FILTER_IP_TOS)
4616                 fconf |= F_TOS;
4617
4618         if (mode & T4_FILTER_VLAN)
4619                 fconf |= F_VLAN;
4620
4621         if (mode & T4_FILTER_VNIC)
4622                 fconf |= F_VNIC_ID;
4623
4624         if (mode & T4_FILTER_PORT)
4625                 fconf |= F_PORT;
4626
4627         if (mode & T4_FILTER_FCoE)
4628                 fconf |= F_FCOE;
4629
4630         return (fconf);
4631 }
4632
4633 static uint32_t
4634 fspec_to_fconf(struct t4_filter_specification *fs)
4635 {
4636         uint32_t fconf = 0;
4637
4638         if (fs->val.frag || fs->mask.frag)
4639                 fconf |= F_FRAGMENTATION;
4640
4641         if (fs->val.matchtype || fs->mask.matchtype)
4642                 fconf |= F_MPSHITTYPE;
4643
4644         if (fs->val.macidx || fs->mask.macidx)
4645                 fconf |= F_MACMATCH;
4646
4647         if (fs->val.ethtype || fs->mask.ethtype)
4648                 fconf |= F_ETHERTYPE;
4649
4650         if (fs->val.proto || fs->mask.proto)
4651                 fconf |= F_PROTOCOL;
4652
4653         if (fs->val.tos || fs->mask.tos)
4654                 fconf |= F_TOS;
4655
4656         if (fs->val.vlan_vld || fs->mask.vlan_vld)
4657                 fconf |= F_VLAN;
4658
4659         if (fs->val.vnic_vld || fs->mask.vnic_vld)
4660                 fconf |= F_VNIC_ID;
4661
4662         if (fs->val.iport || fs->mask.iport)
4663                 fconf |= F_PORT;
4664
4665         if (fs->val.fcoe || fs->mask.fcoe)
4666                 fconf |= F_FCOE;
4667
4668         return (fconf);
4669 }
4670
4671 static int
4672 get_filter_mode(struct adapter *sc, uint32_t *mode)
4673 {
4674         int rc;
4675         uint32_t fconf;
4676
4677         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4678             "t4getfm");
4679         if (rc)
4680                 return (rc);
4681
4682         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
4683             A_TP_VLAN_PRI_MAP);
4684
4685         if (sc->filter_mode != fconf) {
4686                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
4687                     device_get_nameunit(sc->dev), sc->filter_mode, fconf);
4688                 sc->filter_mode = fconf;
4689         }
4690
4691         *mode = fconf_to_mode(sc->filter_mode);
4692
4693         end_synchronized_op(sc, LOCK_HELD);
4694         return (0);
4695 }
4696
4697 static int
4698 set_filter_mode(struct adapter *sc, uint32_t mode)
4699 {
4700         uint32_t fconf;
4701         int rc;
4702
4703         fconf = mode_to_fconf(mode);
4704
4705         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4706             "t4setfm");
4707         if (rc)
4708                 return (rc);
4709
4710         if (sc->tids.ftids_in_use > 0) {
4711                 rc = EBUSY;
4712                 goto done;
4713         }
4714
4715 #ifdef TCP_OFFLOAD
4716         if (sc->offload_map) {
4717                 rc = EBUSY;
4718                 goto done;
4719         }
4720 #endif
4721
4722 #ifdef notyet
4723         rc = -t4_set_filter_mode(sc, fconf);
4724         if (rc == 0)
4725                 sc->filter_mode = fconf;
4726 #else
4727         rc = ENOTSUP;
4728 #endif
4729
4730 done:
4731         end_synchronized_op(sc, LOCK_HELD);
4732         return (rc);
4733 }
4734
4735 static inline uint64_t
4736 get_filter_hits(struct adapter *sc, uint32_t fid)
4737 {
4738         uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4739         uint64_t hits;
4740
4741         t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
4742             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
4743         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
4744         hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
4745
4746         return (be64toh(hits));
4747 }
4748
4749 static int
4750 get_filter(struct adapter *sc, struct t4_filter *t)
4751 {
4752         int i, rc, nfilters = sc->tids.nftids;
4753         struct filter_entry *f;
4754
4755         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4756             "t4getf");
4757         if (rc)
4758                 return (rc);
4759
4760         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
4761             t->idx >= nfilters) {
4762                 t->idx = 0xffffffff;
4763                 goto done;
4764         }
4765
4766         f = &sc->tids.ftid_tab[t->idx];
4767         for (i = t->idx; i < nfilters; i++, f++) {
4768                 if (f->valid) {
4769                         t->idx = i;
4770                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
4771                         t->smtidx = f->smtidx;
4772                         if (f->fs.hitcnts)
4773                                 t->hits = get_filter_hits(sc, t->idx);
4774                         else
4775                                 t->hits = UINT64_MAX;
4776                         t->fs = f->fs;
4777
4778                         goto done;
4779                 }
4780         }
4781
4782         t->idx = 0xffffffff;
4783 done:
4784         end_synchronized_op(sc, LOCK_HELD);
4785         return (0);
4786 }
4787
4788 static int
4789 set_filter(struct adapter *sc, struct t4_filter *t)
4790 {
4791         unsigned int nfilters, nports;
4792         struct filter_entry *f;
4793         int i, rc;
4794
4795         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
4796         if (rc)
4797                 return (rc);
4798
4799         nfilters = sc->tids.nftids;
4800         nports = sc->params.nports;
4801
4802         if (nfilters == 0) {
4803                 rc = ENOTSUP;
4804                 goto done;
4805         }
4806
4807         if (!(sc->flags & FULL_INIT_DONE)) {
4808                 rc = EAGAIN;
4809                 goto done;
4810         }
4811
4812         if (t->idx >= nfilters) {
4813                 rc = EINVAL;
4814                 goto done;
4815         }
4816
4817         /* Validate against the global filter mode */
4818         if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) {
4819                 rc = E2BIG;
4820                 goto done;
4821         }
4822
4823         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
4824                 rc = EINVAL;
4825                 goto done;
4826         }
4827
4828         if (t->fs.val.iport >= nports) {
4829                 rc = EINVAL;
4830                 goto done;
4831         }
4832
4833         /* Can't specify an iq if not steering to it */
4834         if (!t->fs.dirsteer && t->fs.iq) {
4835                 rc = EINVAL;
4836                 goto done;
4837         }
4838
4839         /* IPv6 filter idx must be 4 aligned */
4840         if (t->fs.type == 1 &&
4841             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
4842                 rc = EINVAL;
4843                 goto done;
4844         }
4845
4846         if (sc->tids.ftid_tab == NULL) {
4847                 KASSERT(sc->tids.ftids_in_use == 0,
4848                     ("%s: no memory allocated but filters_in_use > 0",
4849                     __func__));
4850
4851                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
4852                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
4853                 if (sc->tids.ftid_tab == NULL) {
4854                         rc = ENOMEM;
4855                         goto done;
4856                 }
4857                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
4858         }
4859
4860         for (i = 0; i < 4; i++) {
4861                 f = &sc->tids.ftid_tab[t->idx + i];
4862
4863                 if (f->pending || f->valid) {
4864                         rc = EBUSY;
4865                         goto done;
4866                 }
4867                 if (f->locked) {
4868                         rc = EPERM;
4869                         goto done;
4870                 }
4871
4872                 if (t->fs.type == 0)
4873                         break;
4874         }
4875
4876         f = &sc->tids.ftid_tab[t->idx];
4877         f->fs = t->fs;
4878
4879         rc = set_filter_wr(sc, t->idx);
4880 done:
4881         end_synchronized_op(sc, 0);
4882
4883         if (rc == 0) {
4884                 mtx_lock(&sc->tids.ftid_lock);
4885                 for (;;) {
4886                         if (f->pending == 0) {
4887                                 rc = f->valid ? 0 : EIO;
4888                                 break;
4889                         }
4890
4891                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
4892                             PCATCH, "t4setfw", 0)) {
4893                                 rc = EINPROGRESS;
4894                                 break;
4895                         }
4896                 }
4897                 mtx_unlock(&sc->tids.ftid_lock);
4898         }
4899         return (rc);
4900 }
4901
4902 static int
4903 del_filter(struct adapter *sc, struct t4_filter *t)
4904 {
4905         unsigned int nfilters;
4906         struct filter_entry *f;
4907         int rc;
4908
4909         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
4910         if (rc)
4911                 return (rc);
4912
4913         nfilters = sc->tids.nftids;
4914
4915         if (nfilters == 0) {
4916                 rc = ENOTSUP;
4917                 goto done;
4918         }
4919
4920         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
4921             t->idx >= nfilters) {
4922                 rc = EINVAL;
4923                 goto done;
4924         }
4925
4926         if (!(sc->flags & FULL_INIT_DONE)) {
4927                 rc = EAGAIN;
4928                 goto done;
4929         }
4930
4931         f = &sc->tids.ftid_tab[t->idx];
4932
4933         if (f->pending) {
4934                 rc = EBUSY;
4935                 goto done;
4936         }
4937         if (f->locked) {
4938                 rc = EPERM;
4939                 goto done;
4940         }
4941
4942         if (f->valid) {
4943                 t->fs = f->fs;  /* extra info for the caller */
4944                 rc = del_filter_wr(sc, t->idx);
4945         }
4946
4947 done:
4948         end_synchronized_op(sc, 0);
4949
4950         if (rc == 0) {
4951                 mtx_lock(&sc->tids.ftid_lock);
4952                 for (;;) {
4953                         if (f->pending == 0) {
4954                                 rc = f->valid ? EIO : 0;
4955                                 break;
4956                         }
4957
4958                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
4959                             PCATCH, "t4delfw", 0)) {
4960                                 rc = EINPROGRESS;
4961                                 break;
4962                         }
4963                 }
4964                 mtx_unlock(&sc->tids.ftid_lock);
4965         }
4966
4967         return (rc);
4968 }
4969
4970 static void
4971 clear_filter(struct filter_entry *f)
4972 {
4973         if (f->l2t)
4974                 t4_l2t_release(f->l2t);
4975
4976         bzero(f, sizeof (*f));
4977 }
4978
4979 static int
4980 set_filter_wr(struct adapter *sc, int fidx)
4981 {
4982         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4983         struct wrqe *wr;
4984         struct fw_filter_wr *fwr;
4985         unsigned int ftid;
4986
4987         ASSERT_SYNCHRONIZED_OP(sc);
4988
4989         if (f->fs.newdmac || f->fs.newvlan) {
4990                 /* This filter needs an L2T entry; allocate one. */
4991                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
4992                 if (f->l2t == NULL)
4993                         return (EAGAIN);
4994                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
4995                     f->fs.dmac)) {
4996                         t4_l2t_release(f->l2t);
4997                         f->l2t = NULL;
4998                         return (ENOMEM);
4999                 }
5000         }
5001
5002         ftid = sc->tids.ftid_base + fidx;
5003
5004         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5005         if (wr == NULL)
5006                 return (ENOMEM);
5007
5008         fwr = wrtod(wr);
5009         bzero(fwr, sizeof (*fwr));
5010
5011         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
5012         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
5013         fwr->tid_to_iq =
5014             htobe32(V_FW_FILTER_WR_TID(ftid) |
5015                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
5016                 V_FW_FILTER_WR_NOREPLY(0) |
5017                 V_FW_FILTER_WR_IQ(f->fs.iq));
5018         fwr->del_filter_to_l2tix =
5019             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
5020                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
5021                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
5022                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
5023                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
5024                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
5025                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
5026                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
5027                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
5028                     f->fs.newvlan == VLAN_REWRITE) |
5029                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
5030                     f->fs.newvlan == VLAN_REWRITE) |
5031                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
5032                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
5033                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
5034                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
5035         fwr->ethtype = htobe16(f->fs.val.ethtype);
5036         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
5037         fwr->frag_to_ovlan_vldm =
5038             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
5039                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
5040                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
5041                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
5042                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
5043                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
5044         fwr->smac_sel = 0;
5045         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
5046             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
5047         fwr->maci_to_matchtypem =
5048             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
5049                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
5050                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
5051                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
5052                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
5053                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
5054                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
5055                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
5056         fwr->ptcl = f->fs.val.proto;
5057         fwr->ptclm = f->fs.mask.proto;
5058         fwr->ttyp = f->fs.val.tos;
5059         fwr->ttypm = f->fs.mask.tos;
5060         fwr->ivlan = htobe16(f->fs.val.vlan);
5061         fwr->ivlanm = htobe16(f->fs.mask.vlan);
5062         fwr->ovlan = htobe16(f->fs.val.vnic);
5063         fwr->ovlanm = htobe16(f->fs.mask.vnic);
5064         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
5065         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
5066         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
5067         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
5068         fwr->lp = htobe16(f->fs.val.dport);
5069         fwr->lpm = htobe16(f->fs.mask.dport);
5070         fwr->fp = htobe16(f->fs.val.sport);
5071         fwr->fpm = htobe16(f->fs.mask.sport);
5072         if (f->fs.newsmac)
5073                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
5074
5075         f->pending = 1;
5076         sc->tids.ftids_in_use++;
5077
5078         t4_wrq_tx(sc, wr);
5079         return (0);
5080 }
5081
5082 static int
5083 del_filter_wr(struct adapter *sc, int fidx)
5084 {
5085         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
5086         struct wrqe *wr;
5087         struct fw_filter_wr *fwr;
5088         unsigned int ftid;
5089
5090         ftid = sc->tids.ftid_base + fidx;
5091
5092         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5093         if (wr == NULL)
5094                 return (ENOMEM);
5095         fwr = wrtod(wr);
5096         bzero(fwr, sizeof (*fwr));
5097
5098         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
5099
5100         f->pending = 1;
5101         t4_wrq_tx(sc, wr);
5102         return (0);
5103 }
5104
5105 int
5106 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5107 {
5108         struct adapter *sc = iq->adapter;
5109         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
5110         unsigned int idx = GET_TID(rpl);
5111
5112         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5113             rss->opcode));
5114
5115         if (idx >= sc->tids.ftid_base &&
5116             (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
5117                 unsigned int rc = G_COOKIE(rpl->cookie);
5118                 struct filter_entry *f = &sc->tids.ftid_tab[idx];
5119
5120                 mtx_lock(&sc->tids.ftid_lock);
5121                 if (rc == FW_FILTER_WR_FLT_ADDED) {
5122                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
5123                             __func__, idx));
5124                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
5125                         f->pending = 0;  /* asynchronous setup completed */
5126                         f->valid = 1;
5127                 } else {
5128                         if (rc != FW_FILTER_WR_FLT_DELETED) {
5129                                 /* Add or delete failed, display an error */
5130                                 log(LOG_ERR,
5131                                     "filter %u setup failed with error %u\n",
5132                                     idx, rc);
5133                         }
5134
5135                         clear_filter(f);
5136                         sc->tids.ftids_in_use--;
5137                 }
5138                 wakeup(&sc->tids.ftid_tab);
5139                 mtx_unlock(&sc->tids.ftid_lock);
5140         }
5141
5142         return (0);
5143 }
5144
5145 static int
5146 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
5147 {
5148         int rc;
5149
5150         if (cntxt->cid > M_CTXTQID)
5151                 return (EINVAL);
5152
5153         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
5154             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
5155                 return (EINVAL);
5156
5157         if (sc->flags & FW_OK) {
5158                 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4ctxt");
5159                 if (rc == 0) {
5160                         rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid,
5161                             cntxt->mem_id, &cntxt->data[0]);
5162                         end_synchronized_op(sc, LOCK_HELD);
5163                         if (rc == 0)
5164                                 return (0);
5165                 }
5166         }
5167
5168         /*
5169          * Read via firmware failed or wasn't even attempted.  Read directly via
5170          * the backdoor.
5171          */
5172         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
5173             &cntxt->data[0]);
5174         return (rc);
5175 }
5176
5177 static int
5178 load_fw(struct adapter *sc, struct t4_data *fw)
5179 {
5180         int rc;
5181         uint8_t *fw_data;
5182
5183         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
5184         if (rc)
5185                 return (rc);
5186
5187         if (sc->flags & FULL_INIT_DONE) {
5188                 rc = EBUSY;
5189                 goto done;
5190         }
5191
5192         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
5193         if (fw_data == NULL) {
5194                 rc = ENOMEM;
5195                 goto done;
5196         }
5197
5198         rc = copyin(fw->data, fw_data, fw->len);
5199         if (rc == 0)
5200                 rc = -t4_load_fw(sc, fw_data, fw->len);
5201
5202         free(fw_data, M_CXGBE);
5203 done:
5204         end_synchronized_op(sc, 0);
5205         return (rc);
5206 }
5207
5208 static int
5209 read_card_mem(struct adapter *sc, struct t4_mem_range *mr)
5210 {
5211         uint32_t base, size, lo, hi, win, off, remaining, i, n;
5212         uint32_t *buf, *b;
5213         int rc;
5214
5215         /* reads are in multiples of 32 bits */
5216         if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
5217                 return (EINVAL);
5218
5219         /*
5220          * We don't want to deal with potential holes so we mandate that the
5221          * requested region must lie entirely within one of the 3 memories.
5222          */
5223         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5224         if (lo & F_EDRAM0_ENABLE) {
5225                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5226                 base = G_EDRAM0_BASE(hi) << 20;
5227                 size = G_EDRAM0_SIZE(hi) << 20;
5228                 if (size > 0 &&
5229                     mr->addr >= base && mr->addr < base + size &&
5230                     mr->addr + mr->len <= base + size)
5231                         goto proceed;
5232         }
5233         if (lo & F_EDRAM1_ENABLE) {
5234                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5235                 base = G_EDRAM1_BASE(hi) << 20;
5236                 size = G_EDRAM1_SIZE(hi) << 20;
5237                 if (size > 0 &&
5238                     mr->addr >= base && mr->addr < base + size &&
5239                     mr->addr + mr->len <= base + size)
5240                         goto proceed;
5241         }
5242         if (lo & F_EXT_MEM_ENABLE) {
5243                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5244                 base = G_EXT_MEM_BASE(hi) << 20;
5245                 size = G_EXT_MEM_SIZE(hi) << 20;
5246                 if (size > 0 &&
5247                     mr->addr >= base && mr->addr < base + size &&
5248                     mr->addr + mr->len <= base + size)
5249                         goto proceed;
5250         }
5251         return (ENXIO);
5252
5253 proceed:
5254         buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5255
5256         /*
5257          * Position the PCIe window (we use memwin2) to the 16B aligned area
5258          * just at/before the requested region.
5259          */
5260         win = mr->addr & ~0xf;
5261         off = mr->addr - win;  /* offset of the requested region in the win */
5262         remaining = mr->len;
5263
5264         while (remaining) {
5265                 t4_write_reg(sc,
5266                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5267                 t4_read_reg(sc,
5268                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5269
5270                 /* number of bytes that we'll copy in the inner loop */
5271                 n = min(remaining, MEMWIN2_APERTURE - off);
5272
5273                 for (i = 0; i < n; i += 4, remaining -= 4)
5274                         *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5275
5276                 win += MEMWIN2_APERTURE;
5277                 off = 0;
5278         }
5279
5280         rc = copyout(buf, mr->data, mr->len);
5281         free(buf, M_CXGBE);
5282
5283         return (rc);
5284 }
5285
5286 static int
5287 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
5288 {
5289         int rc;
5290
5291         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
5292                 return (EINVAL);
5293
5294         if (i2cd->len > 1) {
5295                 /* XXX: need fw support for longer reads in one go */
5296                 return (ENOTSUP);
5297         }
5298
5299         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
5300         if (rc)
5301                 return (rc);
5302         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
5303             i2cd->offset, &i2cd->data[0]);
5304         end_synchronized_op(sc, 0);
5305
5306         return (rc);
5307 }
5308
5309 int
5310 t4_os_find_pci_capability(struct adapter *sc, int cap)
5311 {
5312         int i;
5313
5314         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5315 }
5316
5317 int
5318 t4_os_pci_save_state(struct adapter *sc)
5319 {
5320         device_t dev;
5321         struct pci_devinfo *dinfo;
5322
5323         dev = sc->dev;
5324         dinfo = device_get_ivars(dev);
5325
5326         pci_cfg_save(dev, dinfo, 0);
5327         return (0);
5328 }
5329
5330 int
5331 t4_os_pci_restore_state(struct adapter *sc)
5332 {
5333         device_t dev;
5334         struct pci_devinfo *dinfo;
5335
5336         dev = sc->dev;
5337         dinfo = device_get_ivars(dev);
5338
5339         pci_cfg_restore(dev, dinfo);
5340         return (0);
5341 }
5342
5343 void
5344 t4_os_portmod_changed(const struct adapter *sc, int idx)
5345 {
5346         struct port_info *pi = sc->port[idx];
5347         static const char *mod_str[] = {
5348                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5349         };
5350
5351         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5352                 if_printf(pi->ifp, "transceiver unplugged.\n");
5353         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5354                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
5355         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5356                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5357         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
5358                 if_printf(pi->ifp, "%s transceiver inserted.\n",
5359                     mod_str[pi->mod_type]);
5360         } else {
5361                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5362                     pi->mod_type);
5363         }
5364 }
5365
5366 void
5367 t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5368 {
5369         struct port_info *pi = sc->port[idx];
5370         struct ifnet *ifp = pi->ifp;
5371
5372         if (link_stat) {
5373                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5374                 if_link_state_change(ifp, LINK_STATE_UP);
5375         } else
5376                 if_link_state_change(ifp, LINK_STATE_DOWN);
5377 }
5378
5379 void
5380 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
5381 {
5382         struct adapter *sc;
5383
5384         mtx_lock(&t4_list_lock);
5385         SLIST_FOREACH(sc, &t4_list, link) {
5386                 /*
5387                  * func should not make any assumptions about what state sc is
5388                  * in - the only guarantee is that sc->sc_lock is a valid lock.
5389                  */
5390                 func(sc, arg);
5391         }
5392         mtx_unlock(&t4_list_lock);
5393 }
5394
5395 static int
5396 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
5397 {
5398        return (0);
5399 }
5400
5401 static int
5402 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
5403 {
5404        return (0);
5405 }
5406
5407 static int
5408 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5409     struct thread *td)
5410 {
5411         int rc;
5412         struct adapter *sc = dev->si_drv1;
5413
5414         rc = priv_check(td, PRIV_DRIVER);
5415         if (rc != 0)
5416                 return (rc);
5417
5418         switch (cmd) {
5419         case CHELSIO_T4_GETREG: {
5420                 struct t4_reg *edata = (struct t4_reg *)data;
5421
5422                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5423                         return (EFAULT);
5424
5425                 if (edata->size == 4)
5426                         edata->val = t4_read_reg(sc, edata->addr);
5427                 else if (edata->size == 8)
5428                         edata->val = t4_read_reg64(sc, edata->addr);
5429                 else
5430                         return (EINVAL);
5431
5432                 break;
5433         }
5434         case CHELSIO_T4_SETREG: {
5435                 struct t4_reg *edata = (struct t4_reg *)data;
5436
5437                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5438                         return (EFAULT);
5439
5440                 if (edata->size == 4) {
5441                         if (edata->val & 0xffffffff00000000)
5442                                 return (EINVAL);
5443                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5444                 } else if (edata->size == 8)
5445                         t4_write_reg64(sc, edata->addr, edata->val);
5446                 else
5447                         return (EINVAL);
5448                 break;
5449         }
5450         case CHELSIO_T4_REGDUMP: {
5451                 struct t4_regdump *regs = (struct t4_regdump *)data;
5452                 int reglen = T4_REGDUMP_SIZE;
5453                 uint8_t *buf;
5454
5455                 if (regs->len < reglen) {
5456                         regs->len = reglen; /* hint to the caller */
5457                         return (ENOBUFS);
5458                 }
5459
5460                 regs->len = reglen;
5461                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5462                 t4_get_regs(sc, regs, buf);
5463                 rc = copyout(buf, regs->data, reglen);
5464                 free(buf, M_CXGBE);
5465                 break;
5466         }
5467         case CHELSIO_T4_GET_FILTER_MODE:
5468                 rc = get_filter_mode(sc, (uint32_t *)data);
5469                 break;
5470         case CHELSIO_T4_SET_FILTER_MODE:
5471                 rc = set_filter_mode(sc, *(uint32_t *)data);
5472                 break;
5473         case CHELSIO_T4_GET_FILTER:
5474                 rc = get_filter(sc, (struct t4_filter *)data);
5475                 break;
5476         case CHELSIO_T4_SET_FILTER:
5477                 rc = set_filter(sc, (struct t4_filter *)data);
5478                 break;
5479         case CHELSIO_T4_DEL_FILTER:
5480                 rc = del_filter(sc, (struct t4_filter *)data);
5481                 break;
5482         case CHELSIO_T4_GET_SGE_CONTEXT:
5483                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
5484                 break;
5485         case CHELSIO_T4_LOAD_FW:
5486                 rc = load_fw(sc, (struct t4_data *)data);
5487                 break;
5488         case CHELSIO_T4_GET_MEM:
5489                 rc = read_card_mem(sc, (struct t4_mem_range *)data);
5490                 break;
5491         case CHELSIO_T4_GET_I2C:
5492                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
5493                 break;
5494         case CHELSIO_T4_CLEAR_STATS: {
5495                 int i;
5496                 u_int port_id = *(uint32_t *)data;
5497                 struct port_info *pi;
5498
5499                 if (port_id >= sc->params.nports)
5500                         return (EINVAL);
5501
5502                 /* MAC stats */
5503                 t4_clr_port_stats(sc, port_id);
5504
5505                 pi = sc->port[port_id];
5506                 if (pi->flags & PORT_INIT_DONE) {
5507                         struct sge_rxq *rxq;
5508                         struct sge_txq *txq;
5509                         struct sge_wrq *wrq;
5510
5511                         for_each_rxq(pi, i, rxq) {
5512 #if defined(INET) || defined(INET6)
5513                                 rxq->lro.lro_queued = 0;
5514                                 rxq->lro.lro_flushed = 0;
5515 #endif
5516                                 rxq->rxcsum = 0;
5517                                 rxq->vlan_extraction = 0;
5518                         }
5519
5520                         for_each_txq(pi, i, txq) {
5521                                 txq->txcsum = 0;
5522                                 txq->tso_wrs = 0;
5523                                 txq->vlan_insertion = 0;
5524                                 txq->imm_wrs = 0;
5525                                 txq->sgl_wrs = 0;
5526                                 txq->txpkt_wrs = 0;
5527                                 txq->txpkts_wrs = 0;
5528                                 txq->txpkts_pkts = 0;
5529                                 txq->no_dmamap = 0;
5530                                 txq->no_desc = 0;
5531                         }
5532
5533 #ifdef TCP_OFFLOAD
5534                         /* nothing to clear for each ofld_rxq */
5535
5536                         for_each_ofld_txq(pi, i, wrq) {
5537                                 wrq->tx_wrs = 0;
5538                                 wrq->no_desc = 0;
5539                         }
5540 #endif
5541                         wrq = &sc->sge.ctrlq[pi->port_id];
5542                         wrq->tx_wrs = 0;
5543                         wrq->no_desc = 0;
5544                 }
5545                 break;
5546         }
5547         default:
5548                 rc = EINVAL;
5549         }
5550
5551         return (rc);
5552 }
5553
5554 #ifdef TCP_OFFLOAD
5555 static int
5556 toe_capability(struct port_info *pi, int enable)
5557 {
5558         int rc;
5559         struct adapter *sc = pi->adapter;
5560
5561         ASSERT_SYNCHRONIZED_OP(sc);
5562
5563         if (!is_offload(sc))
5564                 return (ENODEV);
5565
5566         if (enable) {
5567                 if (!(sc->flags & FULL_INIT_DONE)) {
5568                         rc = cxgbe_init_synchronized(pi);
5569                         if (rc)
5570                                 return (rc);
5571                 }
5572
5573                 if (isset(&sc->offload_map, pi->port_id))
5574                         return (0);
5575
5576                 if (!(sc->flags & TOM_INIT_DONE)) {
5577                         rc = t4_activate_uld(sc, ULD_TOM);
5578                         if (rc == EAGAIN) {
5579                                 log(LOG_WARNING,
5580                                     "You must kldload t4_tom.ko before trying "
5581                                     "to enable TOE on a cxgbe interface.\n");
5582                         }
5583                         if (rc != 0)
5584                                 return (rc);
5585                         KASSERT(sc->tom_softc != NULL,
5586                             ("%s: TOM activated but softc NULL", __func__));
5587                         KASSERT(sc->flags & TOM_INIT_DONE,
5588                             ("%s: TOM activated but flag not set", __func__));
5589                 }
5590
5591                 setbit(&sc->offload_map, pi->port_id);
5592         } else {
5593                 if (!isset(&sc->offload_map, pi->port_id))
5594                         return (0);
5595
5596                 KASSERT(sc->flags & TOM_INIT_DONE,
5597                     ("%s: TOM never initialized?", __func__));
5598                 clrbit(&sc->offload_map, pi->port_id);
5599         }
5600
5601         return (0);
5602 }
5603
5604 /*
5605  * Add an upper layer driver to the global list.
5606  */
5607 int
5608 t4_register_uld(struct uld_info *ui)
5609 {
5610         int rc = 0;
5611         struct uld_info *u;
5612
5613         mtx_lock(&t4_uld_list_lock);
5614         SLIST_FOREACH(u, &t4_uld_list, link) {
5615             if (u->uld_id == ui->uld_id) {
5616                     rc = EEXIST;
5617                     goto done;
5618             }
5619         }
5620
5621         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5622         ui->refcount = 0;
5623 done:
5624         mtx_unlock(&t4_uld_list_lock);
5625         return (rc);
5626 }
5627
5628 int
5629 t4_unregister_uld(struct uld_info *ui)
5630 {
5631         int rc = EINVAL;
5632         struct uld_info *u;
5633
5634         mtx_lock(&t4_uld_list_lock);
5635
5636         SLIST_FOREACH(u, &t4_uld_list, link) {
5637             if (u == ui) {
5638                     if (ui->refcount > 0) {
5639                             rc = EBUSY;
5640                             goto done;
5641                     }
5642
5643                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5644                     rc = 0;
5645                     goto done;
5646             }
5647         }
5648 done:
5649         mtx_unlock(&t4_uld_list_lock);
5650         return (rc);
5651 }
5652
5653 int
5654 t4_activate_uld(struct adapter *sc, int id)
5655 {
5656         int rc = EAGAIN;
5657         struct uld_info *ui;
5658
5659         ASSERT_SYNCHRONIZED_OP(sc);
5660
5661         mtx_lock(&t4_uld_list_lock);
5662
5663         SLIST_FOREACH(ui, &t4_uld_list, link) {
5664                 if (ui->uld_id == id) {
5665                         rc = ui->activate(sc);
5666                         if (rc == 0)
5667                                 ui->refcount++;
5668                         goto done;
5669                 }
5670         }
5671 done:
5672         mtx_unlock(&t4_uld_list_lock);
5673
5674         return (rc);
5675 }
5676
5677 int
5678 t4_deactivate_uld(struct adapter *sc, int id)
5679 {
5680         int rc = EINVAL;
5681         struct uld_info *ui;
5682
5683         ASSERT_SYNCHRONIZED_OP(sc);
5684
5685         mtx_lock(&t4_uld_list_lock);
5686
5687         SLIST_FOREACH(ui, &t4_uld_list, link) {
5688                 if (ui->uld_id == id) {
5689                         rc = ui->deactivate(sc);
5690                         if (rc == 0)
5691                                 ui->refcount--;
5692                         goto done;
5693                 }
5694         }
5695 done:
5696         mtx_unlock(&t4_uld_list_lock);
5697
5698         return (rc);
5699 }
5700 #endif
5701
5702 /*
5703  * Come up with reasonable defaults for some of the tunables, provided they're
5704  * not set by the user (in which case we'll use the values as is).
5705  */
5706 static void
5707 tweak_tunables(void)
5708 {
5709         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
5710
5711         if (t4_ntxq10g < 1)
5712                 t4_ntxq10g = min(nc, NTXQ_10G);
5713
5714         if (t4_ntxq1g < 1)
5715                 t4_ntxq1g = min(nc, NTXQ_1G);
5716
5717         if (t4_nrxq10g < 1)
5718                 t4_nrxq10g = min(nc, NRXQ_10G);
5719
5720         if (t4_nrxq1g < 1)
5721                 t4_nrxq1g = min(nc, NRXQ_1G);
5722
5723 #ifdef TCP_OFFLOAD
5724         if (t4_nofldtxq10g < 1)
5725                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
5726
5727         if (t4_nofldtxq1g < 1)
5728                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
5729
5730         if (t4_nofldrxq10g < 1)
5731                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
5732
5733         if (t4_nofldrxq1g < 1)
5734                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
5735
5736         if (t4_toecaps_allowed == -1)
5737                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
5738 #else
5739         if (t4_toecaps_allowed == -1)
5740                 t4_toecaps_allowed = 0;
5741 #endif
5742
5743         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
5744                 t4_tmr_idx_10g = TMR_IDX_10G;
5745
5746         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
5747                 t4_pktc_idx_10g = PKTC_IDX_10G;
5748
5749         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
5750                 t4_tmr_idx_1g = TMR_IDX_1G;
5751
5752         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
5753                 t4_pktc_idx_1g = PKTC_IDX_1G;
5754
5755         if (t4_qsize_txq < 128)
5756                 t4_qsize_txq = 128;
5757
5758         if (t4_qsize_rxq < 128)
5759                 t4_qsize_rxq = 128;
5760         while (t4_qsize_rxq & 7)
5761                 t4_qsize_rxq++;
5762
5763         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
5764 }
5765
5766 static int
5767 t4_mod_event(module_t mod, int cmd, void *arg)
5768 {
5769         int rc = 0;
5770
5771         switch (cmd) {
5772         case MOD_LOAD:
5773                 t4_sge_modload();
5774                 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
5775                 SLIST_INIT(&t4_list);
5776 #ifdef TCP_OFFLOAD
5777                 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
5778                 SLIST_INIT(&t4_uld_list);
5779 #endif
5780                 tweak_tunables();
5781                 break;
5782
5783         case MOD_UNLOAD:
5784 #ifdef TCP_OFFLOAD
5785                 mtx_lock(&t4_uld_list_lock);
5786                 if (!SLIST_EMPTY(&t4_uld_list)) {
5787                         rc = EBUSY;
5788                         mtx_unlock(&t4_uld_list_lock);
5789                         break;
5790                 }
5791                 mtx_unlock(&t4_uld_list_lock);
5792                 mtx_destroy(&t4_uld_list_lock);
5793 #endif
5794                 mtx_lock(&t4_list_lock);
5795                 if (!SLIST_EMPTY(&t4_list)) {
5796                         rc = EBUSY;
5797                         mtx_unlock(&t4_list_lock);
5798                         break;
5799                 }
5800                 mtx_unlock(&t4_list_lock);
5801                 mtx_destroy(&t4_list_lock);
5802                 break;
5803         }
5804
5805         return (rc);
5806 }
5807
5808 static devclass_t t4_devclass;
5809 static devclass_t cxgbe_devclass;
5810
5811 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
5812 MODULE_VERSION(t4nex, 1);
5813
5814 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
5815 MODULE_VERSION(cxgbe, 1);