sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33 #include "opt_rss.h"
  34
  35 #include <sys/param.h>
  36 #include <sys/conf.h>
  37 #include <sys/priv.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bus.h>
  40 #include <sys/module.h>
  41 #include <sys/malloc.h>
  42 #include <sys/queue.h>
  43 #include <sys/taskqueue.h>
  44 #include <sys/pciio.h>
  45 #include <dev/pci/pcireg.h>
  46 #include <dev/pci/pcivar.h>
  47 #include <dev/pci/pci_private.h>
  48 #include <sys/firmware.h>
  49 #include <sys/sbuf.h>
  50 #include <sys/smp.h>
  51 #include <sys/socket.h>
  52 #include <sys/sockio.h>
  53 #include <sys/sysctl.h>
  54 #include <net/ethernet.h>
  55 #include <net/if.h>
  56 #include <net/if_types.h>
  57 #include <net/if_dl.h>
  58 #include <net/if_vlan_var.h>
  59 #ifdef RSS
  60 #include <net/rss_config.h>
  61 #endif
  62 #if defined(__i386__) || defined(__amd64__)
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65 #endif
  66
  67 #include "common/common.h"
  68 #include "common/t4_msg.h"
  69 #include "common/t4_regs.h"
  70 #include "common/t4_regs_values.h"
  71 #include "t4_ioctl.h"
  72 #include "t4_l2t.h"
  73 #include "t4_mp_ring.h"
  74
  75 /* T4 bus driver interface */
  76 static int t4_probe(device_t);
  77 static int t4_attach(device_t);
  78 static int t4_detach(device_t);
  79 static device_method_t t4_methods[] = {
  80         DEVMETHOD(device_probe,         t4_probe),
  81         DEVMETHOD(device_attach,        t4_attach),
  82         DEVMETHOD(device_detach,        t4_detach),
  83
  84         DEVMETHOD_END
  85 };
  86 static driver_t t4_driver = {
  87         "t4nex",
  88         t4_methods,
  89         sizeof(struct adapter)
  90 };
  91
  92
  93 /* T4 port (cxgbe) interface */
  94 static int cxgbe_probe(device_t);
  95 static int cxgbe_attach(device_t);
  96 static int cxgbe_detach(device_t);
  97 static device_method_t cxgbe_methods[] = {
  98         DEVMETHOD(device_probe,         cxgbe_probe),
  99         DEVMETHOD(device_attach,        cxgbe_attach),
 100         DEVMETHOD(device_detach,        cxgbe_detach),
 101         { 0, 0 }
 102 };
 103 static driver_t cxgbe_driver = {
 104         "cxgbe",
 105         cxgbe_methods,
 106         sizeof(struct port_info)
 107 };
 108
 109 /* T4 VI (vcxgbe) interface */
 110 static int vcxgbe_probe(device_t);
 111 static int vcxgbe_attach(device_t);
 112 static int vcxgbe_detach(device_t);
 113 static device_method_t vcxgbe_methods[] = {
 114         DEVMETHOD(device_probe,         vcxgbe_probe),
 115         DEVMETHOD(device_attach,        vcxgbe_attach),
 116         DEVMETHOD(device_detach,        vcxgbe_detach),
 117         { 0, 0 }
 118 };
 119 static driver_t vcxgbe_driver = {
 120         "vcxgbe",
 121         vcxgbe_methods,
 122         sizeof(struct vi_info)
 123 };
 124
 125 static d_ioctl_t t4_ioctl;
 126 static d_open_t t4_open;
 127 static d_close_t t4_close;
 128
 129 static struct cdevsw t4_cdevsw = {
 130        .d_version = D_VERSION,
 131        .d_flags = 0,
 132        .d_open = t4_open,
 133        .d_close = t4_close,
 134        .d_ioctl = t4_ioctl,
 135        .d_name = "t4nex",
 136 };
 137
 138 /* T5 bus driver interface */
 139 static int t5_probe(device_t);
 140 static device_method_t t5_methods[] = {
 141         DEVMETHOD(device_probe,         t5_probe),
 142         DEVMETHOD(device_attach,        t4_attach),
 143         DEVMETHOD(device_detach,        t4_detach),
 144
 145         DEVMETHOD_END
 146 };
 147 static driver_t t5_driver = {
 148         "t5nex",
 149         t5_methods,
 150         sizeof(struct adapter)
 151 };
 152
 153
 154 /* T5 port (cxl) interface */
 155 static driver_t cxl_driver = {
 156         "cxl",
 157         cxgbe_methods,
 158         sizeof(struct port_info)
 159 };
 160
 161 /* T5 VI (vcxl) interface */
 162 static driver_t vcxl_driver = {
 163         "vcxl",
 164         vcxgbe_methods,
 165         sizeof(struct vi_info)
 166 };
 167
 168 static struct cdevsw t5_cdevsw = {
 169        .d_version = D_VERSION,
 170        .d_flags = 0,
 171        .d_open = t4_open,
 172        .d_close = t4_close,
 173        .d_ioctl = t4_ioctl,
 174        .d_name = "t5nex",
 175 };
 176
 177 /* ifnet + media interface */
 178 static void cxgbe_init(void *);
 179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 180 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 181 static void cxgbe_qflush(struct ifnet *);
 182 static int cxgbe_media_change(struct ifnet *);
 183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 184
 185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 186
 187 /*
 188  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 189  * then ADAPTER_LOCK, then t4_uld_list_lock.
 190  */
 191 static struct sx t4_list_lock;
 192 SLIST_HEAD(, adapter) t4_list;
 193 #ifdef TCP_OFFLOAD
 194 static struct sx t4_uld_list_lock;
 195 SLIST_HEAD(, uld_info) t4_uld_list;
 196 #endif
 197
 198 /*
 199  * Tunables.  See tweak_tunables() too.
 200  *
 201  * Each tunable is set to a default value here if it's known at compile-time.
 202  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 203  * provide a reasonable default when the driver is loaded.
 204  *
 205  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 206  * T5 are under hw.cxl.
 207  */
 208
 209 /*
 210  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 211  */
 212 #define NTXQ_10G 16
 213 static int t4_ntxq10g = -1;
 214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 215
 216 #define NRXQ_10G 8
 217 static int t4_nrxq10g = -1;
 218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 219
 220 #define NTXQ_1G 4
 221 static int t4_ntxq1g = -1;
 222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 223
 224 #define NRXQ_1G 2
 225 static int t4_nrxq1g = -1;
 226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 227
 228 static int t4_rsrv_noflowq = 0;
 229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 230
 231 #ifdef TCP_OFFLOAD
 232 #define NOFLDTXQ_10G 8
 233 static int t4_nofldtxq10g = -1;
 234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 235
 236 #define NOFLDRXQ_10G 2
 237 static int t4_nofldrxq10g = -1;
 238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 239
 240 #define NOFLDTXQ_1G 2
 241 static int t4_nofldtxq1g = -1;
 242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 243
 244 #define NOFLDRXQ_1G 1
 245 static int t4_nofldrxq1g = -1;
 246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 247 #endif
 248
 249 #ifdef DEV_NETMAP
 250 #define NNMTXQ_10G 2
 251 static int t4_nnmtxq10g = -1;
 252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 253
 254 #define NNMRXQ_10G 2
 255 static int t4_nnmrxq10g = -1;
 256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 257
 258 #define NNMTXQ_1G 1
 259 static int t4_nnmtxq1g = -1;
 260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 261
 262 #define NNMRXQ_1G 1
 263 static int t4_nnmrxq1g = -1;
 264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 265 #endif
 266
 267 /*
 268  * Holdoff parameters for 10G and 1G ports.
 269  */
 270 #define TMR_IDX_10G 1
 271 static int t4_tmr_idx_10g = TMR_IDX_10G;
 272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 273
 274 #define PKTC_IDX_10G (-1)
 275 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 277
 278 #define TMR_IDX_1G 1
 279 static int t4_tmr_idx_1g = TMR_IDX_1G;
 280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 281
 282 #define PKTC_IDX_1G (-1)
 283 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 285
 286 /*
 287  * Size (# of entries) of each tx and rx queue.
 288  */
 289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 291
 292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 294
 295 /*
 296  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 297  */
 298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 300
 301 /*
 302  * Configuration file.
 303  */
 304 #define DEFAULT_CF      "default"
 305 #define FLASH_CF        "flash"
 306 #define UWIRE_CF        "uwire"
 307 #define FPGA_CF         "fpga"
 308 static char t4_cfg_file[32] = DEFAULT_CF;
 309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 310
 311 /*
 312  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 313  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 314  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 315  *            mark or when signalled to do so, 0 to never emit PAUSE.
 316  */
 317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 319
 320 /*
 321  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 322  * encouraged respectively).
 323  */
 324 static unsigned int t4_fw_install = 1;
 325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 326
 327 /*
 328  * ASIC features that will be used.  Disable the ones you don't want so that the
 329  * chip resources aren't wasted on features that will not be used.
 330  */
 331 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 333
 334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 336
 337 static int t4_toecaps_allowed = -1;
 338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 339
 340 static int t4_rdmacaps_allowed = 0;
 341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 342
 343 static int t4_iscsicaps_allowed = 0;
 344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 345
 346 static int t4_fcoecaps_allowed = 0;
 347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 348
 349 static int t5_write_combine = 0;
 350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 351
 352 static int t4_num_vis = 1;
 353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
 354
 355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */
 356 static int vi_mac_funcs[] = {
 357         FW_VI_FUNC_OFLD,
 358         FW_VI_FUNC_IWARP,
 359         FW_VI_FUNC_OPENISCSI,
 360         FW_VI_FUNC_OPENFCOE,
 361         FW_VI_FUNC_FOISCSI,
 362         FW_VI_FUNC_FOFCOE,
 363 };
 364
 365 struct intrs_and_queues {
 366         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 367         uint16_t nirq;          /* Total # of vectors */
 368         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 369         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 370         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 371         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 372         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 373         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 374         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 375 #ifdef TCP_OFFLOAD
 376         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 377         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 378         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 379         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 380 #endif
 381 #ifdef DEV_NETMAP
 382         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 383         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 384         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 385         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 386 #endif
 387 };
 388
 389 struct filter_entry {
 390         uint32_t valid:1;       /* filter allocated and valid */
 391         uint32_t locked:1;      /* filter is administratively locked */
 392         uint32_t pending:1;     /* filter action is pending firmware reply */
 393         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 394         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 395
 396         struct t4_filter_specification fs;
 397 };
 398
 399 static int map_bars_0_and_4(struct adapter *);
 400 static int map_bar_2(struct adapter *);
 401 static void setup_memwin(struct adapter *);
 402 static int validate_mem_range(struct adapter *, uint32_t, int);
 403 static int fwmtype_to_hwmtype(int);
 404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 405     uint32_t *);
 406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 407 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int,
 409     struct intrs_and_queues *);
 410 static int prep_firmware(struct adapter *);
 411 static int partition_resources(struct adapter *, const struct firmware *,
 412     const char *);
 413 static int get_params__pre_init(struct adapter *);
 414 static int get_params__post_init(struct adapter *);
 415 static int set_params__post_init(struct adapter *);
 416 static void t4_set_desc(struct adapter *);
 417 static void build_medialist(struct port_info *, struct ifmedia *);
 418 static int cxgbe_init_synchronized(struct vi_info *);
 419 static int cxgbe_uninit_synchronized(struct vi_info *);
 420 static int setup_intr_handlers(struct adapter *);
 421 static void quiesce_txq(struct adapter *, struct sge_txq *);
 422 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 423 static void quiesce_iq(struct adapter *, struct sge_iq *);
 424 static void quiesce_fl(struct adapter *, struct sge_fl *);
 425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 426     driver_intr_t *, void *, char *);
 427 static int t4_free_irq(struct adapter *, struct irq *);
 428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 429     unsigned int);
 430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 431 static void vi_refresh_stats(struct adapter *, struct vi_info *);
 432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 433 static void cxgbe_tick(void *);
 434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 436     struct mbuf *);
 437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 438 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 439 static void t4_sysctls(struct adapter *);
 440 static void cxgbe_sysctls(struct port_info *);
 441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 452 #ifdef SBUF_DRAIN
 453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 472 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 478 #endif
 479 static uint32_t fconf_to_mode(uint32_t);
 480 static uint32_t mode_to_fconf(uint32_t);
 481 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 482 static int get_filter_mode(struct adapter *, uint32_t *);
 483 static int set_filter_mode(struct adapter *, uint32_t);
 484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 485 static int get_filter(struct adapter *, struct t4_filter *);
 486 static int set_filter(struct adapter *, struct t4_filter *);
 487 static int del_filter(struct adapter *, struct t4_filter *);
 488 static void clear_filter(struct filter_entry *);
 489 static int set_filter_wr(struct adapter *, int);
 490 static int del_filter_wr(struct adapter *, int);
 491 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 492 static int load_fw(struct adapter *, struct t4_data *);
 493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 494 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 495 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 497 #ifdef TCP_OFFLOAD
 498 static int toe_capability(struct vi_info *, int);
 499 #endif
 500 static int mod_event(module_t, int, void *);
 501
 502 struct {
 503         uint16_t device;
 504         char *desc;
 505 } t4_pciids[] = {
 506         {0xa000, "Chelsio Terminator 4 FPGA"},
 507         {0x4400, "Chelsio T440-dbg"},
 508         {0x4401, "Chelsio T420-CR"},
 509         {0x4402, "Chelsio T422-CR"},
 510         {0x4403, "Chelsio T440-CR"},
 511         {0x4404, "Chelsio T420-BCH"},
 512         {0x4405, "Chelsio T440-BCH"},
 513         {0x4406, "Chelsio T440-CH"},
 514         {0x4407, "Chelsio T420-SO"},
 515         {0x4408, "Chelsio T420-CX"},
 516         {0x4409, "Chelsio T420-BT"},
 517         {0x440a, "Chelsio T404-BT"},
 518         {0x440e, "Chelsio T440-LP-CR"},
 519 }, t5_pciids[] = {
 520         {0xb000, "Chelsio Terminator 5 FPGA"},
 521         {0x5400, "Chelsio T580-dbg"},
 522         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 523         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 524         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 525         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 526         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 527         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 528         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 529         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 530         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 531         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 532         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 533         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 534         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 535 #ifdef notyet
 536         {0x5404,  "Chelsio T520-BCH"},
 537         {0x5405,  "Chelsio T540-BCH"},
 538         {0x5406,  "Chelsio T540-CH"},
 539         {0x5408,  "Chelsio T520-CX"},
 540         {0x540b,  "Chelsio B520-SR"},
 541         {0x540c,  "Chelsio B504-BT"},
 542         {0x540f,  "Chelsio Amsterdam"},
 543         {0x5413,  "Chelsio T580-CHR"},
 544 #endif
 545 };
 546
 547 #ifdef TCP_OFFLOAD
 548 /*
 549  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 550  * exactly the same for both rxq and ofld_rxq.
 551  */
 552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 554 #endif
 555
 556 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 559
 560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 561
 562 static int
 563 t4_probe(device_t dev)
 564 {
 565         int i;
 566         uint16_t v = pci_get_vendor(dev);
 567         uint16_t d = pci_get_device(dev);
 568         uint8_t f = pci_get_function(dev);
 569
 570         if (v != PCI_VENDOR_ID_CHELSIO)
 571                 return (ENXIO);
 572
 573         /* Attach only to PF0 of the FPGA */
 574         if (d == 0xa000 && f != 0)
 575                 return (ENXIO);
 576
 577         for (i = 0; i < nitems(t4_pciids); i++) {
 578                 if (d == t4_pciids[i].device) {
 579                         device_set_desc(dev, t4_pciids[i].desc);
 580                         return (BUS_PROBE_DEFAULT);
 581                 }
 582         }
 583
 584         return (ENXIO);
 585 }
 586
 587 static int
 588 t5_probe(device_t dev)
 589 {
 590         int i;
 591         uint16_t v = pci_get_vendor(dev);
 592         uint16_t d = pci_get_device(dev);
 593         uint8_t f = pci_get_function(dev);
 594
 595         if (v != PCI_VENDOR_ID_CHELSIO)
 596                 return (ENXIO);
 597
 598         /* Attach only to PF0 of the FPGA */
 599         if (d == 0xb000 && f != 0)
 600                 return (ENXIO);
 601
 602         for (i = 0; i < nitems(t5_pciids); i++) {
 603                 if (d == t5_pciids[i].device) {
 604                         device_set_desc(dev, t5_pciids[i].desc);
 605                         return (BUS_PROBE_DEFAULT);
 606                 }
 607         }
 608
 609         return (ENXIO);
 610 }
 611
 612 static void
 613 t5_attribute_workaround(device_t dev)
 614 {
 615         device_t root_port;
 616         uint32_t v;
 617
 618         /*
 619          * The T5 chips do not properly echo the No Snoop and Relaxed
 620          * Ordering attributes when replying to a TLP from a Root
 621          * Port.  As a workaround, find the parent Root Port and
 622          * disable No Snoop and Relaxed Ordering.  Note that this
 623          * affects all devices under this root port.
 624          */
 625         root_port = pci_find_pcie_root_port(dev);
 626         if (root_port == NULL) {
 627                 device_printf(dev, "Unable to find parent root port\n");
 628                 return;
 629         }
 630
 631         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 632             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 633         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 634             0)
 635                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 636                     device_get_nameunit(root_port));
 637 }
 638
 639 static int
 640 t4_attach(device_t dev)
 641 {
 642         struct adapter *sc;
 643         int rc = 0, i, j, n10g, n1g, rqidx, tqidx;
 644         struct intrs_and_queues iaq;
 645         struct sge *s;
 646 #ifdef TCP_OFFLOAD
 647         int ofld_rqidx, ofld_tqidx;
 648 #endif
 649 #ifdef DEV_NETMAP
 650         int nm_rqidx, nm_tqidx;
 651 #endif
 652         int num_vis;
 653
 654         sc = device_get_softc(dev);
 655         sc->dev = dev;
 656         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 657
 658         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 659                 t5_attribute_workaround(dev);
 660         pci_enable_busmaster(dev);
 661         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 662                 uint32_t v;
 663
 664                 pci_set_max_read_req(dev, 4096);
 665                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 666                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 667                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 668
 669                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 670         }
 671
 672         sc->traceq = -1;
 673         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 674         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 675             device_get_nameunit(dev));
 676
 677         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 678             device_get_nameunit(dev));
 679         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 680         sx_xlock(&t4_list_lock);
 681         SLIST_INSERT_HEAD(&t4_list, sc, link);
 682         sx_xunlock(&t4_list_lock);
 683
 684         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 685         TAILQ_INIT(&sc->sfl);
 686         callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
 687
 688         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 689
 690         rc = map_bars_0_and_4(sc);
 691         if (rc != 0)
 692                 goto done; /* error message displayed already */
 693
 694         /*
 695          * This is the real PF# to which we're attaching.  Works from within PCI
 696          * passthrough environments too, where pci_get_function() could return a
 697          * different PF# depending on the passthrough configuration.  We need to
 698          * use the real PF# in all our communication with the firmware.
 699          */
 700         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 701         sc->mbox = sc->pf;
 702
 703         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 704         sc->an_handler = an_not_handled;
 705         for (i = 0; i < nitems(sc->cpl_handler); i++)
 706                 sc->cpl_handler[i] = cpl_not_handled;
 707         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 708                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 709         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 710         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 711         t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt);
 712         t4_init_sge_cpl_handlers(sc);
 713
 714         /* Prepare the adapter for operation */
 715         rc = -t4_prep_adapter(sc);
 716         if (rc != 0) {
 717                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 718                 goto done;
 719         }
 720
 721         /*
 722          * Do this really early, with the memory windows set up even before the
 723          * character device.  The userland tool's register i/o and mem read
 724          * will work even in "recovery mode".
 725          */
 726         setup_memwin(sc);
 727         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 728             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 729             device_get_nameunit(dev));
 730         if (sc->cdev == NULL)
 731                 device_printf(dev, "failed to create nexus char device.\n");
 732         else
 733                 sc->cdev->si_drv1 = sc;
 734
 735         /* Go no further if recovery mode has been requested. */
 736         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 737                 device_printf(dev, "recovery mode.\n");
 738                 goto done;
 739         }
 740
 741 #if defined(__i386__)
 742         if ((cpu_feature & CPUID_CX8) == 0) {
 743                 device_printf(dev, "64 bit atomics not available.\n");
 744                 rc = ENOTSUP;
 745                 goto done;
 746         }
 747 #endif
 748
 749         /* Prepare the firmware for operation */
 750         rc = prep_firmware(sc);
 751         if (rc != 0)
 752                 goto done; /* error message displayed already */
 753
 754         rc = get_params__post_init(sc);
 755         if (rc != 0)
 756                 goto done; /* error message displayed already */
 757
 758         rc = set_params__post_init(sc);
 759         if (rc != 0)
 760                 goto done; /* error message displayed already */
 761
 762         rc = map_bar_2(sc);
 763         if (rc != 0)
 764                 goto done; /* error message displayed already */
 765
 766         rc = t4_create_dma_tag(sc);
 767         if (rc != 0)
 768                 goto done; /* error message displayed already */
 769
 770         /*
 771          * Number of VIs to create per-port.  The first VI is the
 772          * "main" regular VI for the port.  The second VI is used for
 773          * netmap if present, and any remaining VIs are used for
 774          * additional virtual interfaces.
 775          *
 776          * Limit the number of VIs per port to the number of available
 777          * MAC addresses per port.
 778          */
 779         if (t4_num_vis >= 1)
 780                 num_vis = t4_num_vis;
 781         else
 782                 num_vis = 1;
 783 #ifdef DEV_NETMAP
 784         num_vis++;
 785 #endif
 786         if (num_vis > nitems(vi_mac_funcs)) {
 787                 num_vis = nitems(vi_mac_funcs);
 788                 device_printf(dev, "Number of VIs limited to %d\n", num_vis);
 789         }
 790
 791         /*
 792          * First pass over all the ports - allocate VIs and initialize some
 793          * basic parameters like mac address, port type, etc.  We also figure
 794          * out whether a port is 10G or 1G and use that information when
 795          * calculating how many interrupts to attempt to allocate.
 796          */
 797         n10g = n1g = 0;
 798         for_each_port(sc, i) {
 799                 struct port_info *pi;
 800                 struct vi_info *vi;
 801
 802                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 803                 sc->port[i] = pi;
 804
 805                 /* These must be set before t4_port_init */
 806                 pi->adapter = sc;
 807                 pi->port_id = i;
 808                 pi->nvi = num_vis;
 809                 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE,
 810                     M_ZERO | M_WAITOK);
 811
 812                 /*
 813                  * Allocate the "main" VI and initialize parameters
 814                  * like mac addr.
 815                  */
 816                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 817                 if (rc != 0) {
 818                         device_printf(dev, "unable to initialize port %d: %d\n",
 819                             i, rc);
 820                         free(pi->vi, M_CXGBE);
 821                         free(pi, M_CXGBE);
 822                         sc->port[i] = NULL;
 823                         goto done;
 824                 }
 825
 826                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 827                 pi->link_cfg.requested_fc |= t4_pause_settings;
 828                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 829                 pi->link_cfg.fc |= t4_pause_settings;
 830
 831                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 832                 if (rc != 0) {
 833                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 834                         free(pi->vi, M_CXGBE);
 835                         free(pi, M_CXGBE);
 836                         sc->port[i] = NULL;
 837                         goto done;
 838                 }
 839
 840                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 841                     device_get_nameunit(dev), i);
 842                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 843                 sc->chan_map[pi->tx_chan] = i;
 844
 845                 if (is_10G_port(pi) || is_40G_port(pi)) {
 846                         n10g++;
 847                         for_each_vi(pi, j, vi) {
 848                                 vi->tmr_idx = t4_tmr_idx_10g;
 849                                 vi->pktc_idx = t4_pktc_idx_10g;
 850                         }
 851                 } else {
 852                         n1g++;
 853                         for_each_vi(pi, j, vi) {
 854                                 vi->tmr_idx = t4_tmr_idx_1g;
 855                                 vi->pktc_idx = t4_pktc_idx_1g;
 856                         }
 857                 }
 858
 859                 pi->linkdnrc = -1;
 860
 861                 for_each_vi(pi, j, vi) {
 862                         vi->qsize_rxq = t4_qsize_rxq;
 863                         vi->qsize_txq = t4_qsize_txq;
 864                         vi->pi = pi;
 865                 }
 866
 867                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 868                 if (pi->dev == NULL) {
 869                         device_printf(dev,
 870                             "failed to add device for port %d.\n", i);
 871                         rc = ENXIO;
 872                         goto done;
 873                 }
 874                 pi->vi[0].dev = pi->dev;
 875                 device_set_softc(pi->dev, pi);
 876         }
 877
 878         /*
 879          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 880          */
 881 #ifdef DEV_NETMAP
 882         num_vis--;
 883 #endif
 884         rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq);
 885         if (rc != 0)
 886                 goto done; /* error message displayed already */
 887
 888         sc->intr_type = iaq.intr_type;
 889         sc->intr_count = iaq.nirq;
 890
 891         s = &sc->sge;
 892         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 893         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 894         if (num_vis > 1) {
 895                 s->nrxq += (n10g + n1g) * (num_vis - 1);
 896                 s->ntxq += (n10g + n1g) * (num_vis - 1);
 897         }
 898         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 899         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 900         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 901 #ifdef TCP_OFFLOAD
 902         if (is_offload(sc)) {
 903                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 904                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 905                 if (num_vis > 1) {
 906                         s->nofldrxq += (n10g + n1g) * (num_vis - 1);
 907                         s->nofldtxq += (n10g + n1g) * (num_vis - 1);
 908                 }
 909                 s->neq += s->nofldtxq + s->nofldrxq;
 910                 s->niq += s->nofldrxq;
 911
 912                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 913                     M_CXGBE, M_ZERO | M_WAITOK);
 914                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 915                     M_CXGBE, M_ZERO | M_WAITOK);
 916         }
 917 #endif
 918 #ifdef DEV_NETMAP
 919         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 920         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 921         s->neq += s->nnmtxq + s->nnmrxq;
 922         s->niq += s->nnmrxq;
 923
 924         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 925             M_CXGBE, M_ZERO | M_WAITOK);
 926         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 927             M_CXGBE, M_ZERO | M_WAITOK);
 928 #endif
 929
 930         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 931             M_ZERO | M_WAITOK);
 932         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 933             M_ZERO | M_WAITOK);
 934         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 935             M_ZERO | M_WAITOK);
 936         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 937             M_ZERO | M_WAITOK);
 938         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 939             M_ZERO | M_WAITOK);
 940
 941         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 942             M_ZERO | M_WAITOK);
 943
 944         t4_init_l2t(sc, M_WAITOK);
 945
 946         /*
 947          * Second pass over the ports.  This time we know the number of rx and
 948          * tx queues that each port should get.
 949          */
 950         rqidx = tqidx = 0;
 951 #ifdef TCP_OFFLOAD
 952         ofld_rqidx = ofld_tqidx = 0;
 953 #endif
 954 #ifdef DEV_NETMAP
 955         nm_rqidx = nm_tqidx = 0;
 956 #endif
 957         for_each_port(sc, i) {
 958                 struct port_info *pi = sc->port[i];
 959                 struct vi_info *vi;
 960
 961                 if (pi == NULL)
 962                         continue;
 963
 964                 for_each_vi(pi, j, vi) {
 965 #ifdef DEV_NETMAP
 966                         if (j == 1) {
 967                                 vi->flags |= VI_NETMAP | INTR_RXQ;
 968                                 vi->first_rxq = nm_rqidx;
 969                                 vi->first_txq = nm_tqidx;
 970                                 if (is_10G_port(pi) || is_40G_port(pi)) {
 971                                         vi->nrxq = iaq.nnmrxq10g;
 972                                         vi->ntxq = iaq.nnmtxq10g;
 973                                 } else {
 974                                         vi->nrxq = iaq.nnmrxq1g;
 975                                         vi->ntxq = iaq.nnmtxq1g;
 976                                 }
 977                                 nm_rqidx += vi->nrxq;
 978                                 nm_tqidx += vi->ntxq;
 979                                 continue;
 980                         }
 981 #endif
 982
 983                         vi->first_rxq = rqidx;
 984                         vi->first_txq = tqidx;
 985                         if (is_10G_port(pi) || is_40G_port(pi)) {
 986                                 vi->flags |= iaq.intr_flags_10g & INTR_RXQ;
 987                                 vi->nrxq = j == 0 ? iaq.nrxq10g : 1;
 988                                 vi->ntxq = j == 0 ? iaq.ntxq10g : 1;
 989                         } else {
 990                                 vi->flags |= iaq.intr_flags_1g & INTR_RXQ;
 991                                 vi->nrxq = j == 0 ? iaq.nrxq1g : 1;
 992                                 vi->ntxq = j == 0 ? iaq.ntxq1g : 1;
 993                         }
 994
 995                         if (vi->ntxq > 1)
 996                                 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 997                         else
 998                                 vi->rsrv_noflowq = 0;
 999
1000                         rqidx += vi->nrxq;
1001                         tqidx += vi->ntxq;
1002
1003 #ifdef TCP_OFFLOAD
1004                         if (!is_offload(sc))
1005                                 continue;
1006                         vi->first_ofld_rxq = ofld_rqidx;
1007                         vi->first_ofld_txq = ofld_tqidx;
1008                         if (is_10G_port(pi) || is_40G_port(pi)) {
1009                                 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ;
1010                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1;
1011                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1;
1012                         } else {
1013                                 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ;
1014                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1;
1015                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1;
1016                         }
1017                         ofld_rqidx += vi->nofldrxq;
1018                         ofld_tqidx += vi->nofldtxq;
1019 #endif
1020                 }
1021         }
1022
1023         rc = setup_intr_handlers(sc);
1024         if (rc != 0) {
1025                 device_printf(dev,
1026                     "failed to setup interrupt handlers: %d\n", rc);
1027                 goto done;
1028         }
1029
1030         rc = bus_generic_attach(dev);
1031         if (rc != 0) {
1032                 device_printf(dev,
1033                     "failed to attach all child ports: %d\n", rc);
1034                 goto done;
1035         }
1036
1037         device_printf(dev,
1038             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1039             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1040             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1041             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1042             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1043
1044         t4_set_desc(sc);
1045
1046 done:
1047         if (rc != 0 && sc->cdev) {
1048                 /* cdev was created and so cxgbetool works; recover that way. */
1049                 device_printf(dev,
1050                     "error during attach, adapter is now in recovery mode.\n");
1051                 rc = 0;
1052         }
1053
1054         if (rc != 0)
1055                 t4_detach(dev);
1056         else
1057                 t4_sysctls(sc);
1058
1059         return (rc);
1060 }
1061
1062 /*
1063  * Idempotent
1064  */
1065 static int
1066 t4_detach(device_t dev)
1067 {
1068         struct adapter *sc;
1069         struct port_info *pi;
1070         int i, rc;
1071
1072         sc = device_get_softc(dev);
1073
1074         if (sc->flags & FULL_INIT_DONE)
1075                 t4_intr_disable(sc);
1076
1077         if (sc->cdev) {
1078                 destroy_dev(sc->cdev);
1079                 sc->cdev = NULL;
1080         }
1081
1082         rc = bus_generic_detach(dev);
1083         if (rc) {
1084                 device_printf(dev,
1085                     "failed to detach child devices: %d\n", rc);
1086                 return (rc);
1087         }
1088
1089         for (i = 0; i < sc->intr_count; i++)
1090                 t4_free_irq(sc, &sc->irq[i]);
1091
1092         for (i = 0; i < MAX_NPORTS; i++) {
1093                 pi = sc->port[i];
1094                 if (pi) {
1095                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1096                         if (pi->dev)
1097                                 device_delete_child(dev, pi->dev);
1098
1099                         mtx_destroy(&pi->pi_lock);
1100                         free(pi->vi, M_CXGBE);
1101                         free(pi, M_CXGBE);
1102                 }
1103         }
1104
1105         if (sc->flags & FULL_INIT_DONE)
1106                 adapter_full_uninit(sc);
1107
1108         if (sc->flags & FW_OK)
1109                 t4_fw_bye(sc, sc->mbox);
1110
1111         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1112                 pci_release_msi(dev);
1113
1114         if (sc->regs_res)
1115                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1116                     sc->regs_res);
1117
1118         if (sc->udbs_res)
1119                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1120                     sc->udbs_res);
1121
1122         if (sc->msix_res)
1123                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1124                     sc->msix_res);
1125
1126         if (sc->l2t)
1127                 t4_free_l2t(sc->l2t);
1128
1129 #ifdef TCP_OFFLOAD
1130         free(sc->sge.ofld_rxq, M_CXGBE);
1131         free(sc->sge.ofld_txq, M_CXGBE);
1132 #endif
1133 #ifdef DEV_NETMAP
1134         free(sc->sge.nm_rxq, M_CXGBE);
1135         free(sc->sge.nm_txq, M_CXGBE);
1136 #endif
1137         free(sc->irq, M_CXGBE);
1138         free(sc->sge.rxq, M_CXGBE);
1139         free(sc->sge.txq, M_CXGBE);
1140         free(sc->sge.ctrlq, M_CXGBE);
1141         free(sc->sge.iqmap, M_CXGBE);
1142         free(sc->sge.eqmap, M_CXGBE);
1143         free(sc->tids.ftid_tab, M_CXGBE);
1144         t4_destroy_dma_tag(sc);
1145         if (mtx_initialized(&sc->sc_lock)) {
1146                 sx_xlock(&t4_list_lock);
1147                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1148                 sx_xunlock(&t4_list_lock);
1149                 mtx_destroy(&sc->sc_lock);
1150         }
1151
1152         callout_drain(&sc->sfl_callout);
1153         if (mtx_initialized(&sc->tids.ftid_lock))
1154                 mtx_destroy(&sc->tids.ftid_lock);
1155         if (mtx_initialized(&sc->sfl_lock))
1156                 mtx_destroy(&sc->sfl_lock);
1157         if (mtx_initialized(&sc->ifp_lock))
1158                 mtx_destroy(&sc->ifp_lock);
1159         if (mtx_initialized(&sc->regwin_lock))
1160                 mtx_destroy(&sc->regwin_lock);
1161
1162         bzero(sc, sizeof(*sc));
1163
1164         return (0);
1165 }
1166
1167 static int
1168 cxgbe_probe(device_t dev)
1169 {
1170         char buf[128];
1171         struct port_info *pi = device_get_softc(dev);
1172
1173         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1174         device_set_desc_copy(dev, buf);
1175
1176         return (BUS_PROBE_DEFAULT);
1177 }
1178
1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1180     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1181     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1182 #define T4_CAP_ENABLE (T4_CAP)
1183
1184 static int
1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1186 {
1187         struct ifnet *ifp;
1188         struct sbuf *sb;
1189
1190         vi->xact_addr_filt = -1;
1191         callout_init(&vi->tick, 1);
1192
1193         /* Allocate an ifnet and set it up */
1194         ifp = if_alloc(IFT_ETHER);
1195         if (ifp == NULL) {
1196                 device_printf(dev, "Cannot allocate ifnet\n");
1197                 return (ENOMEM);
1198         }
1199         vi->ifp = ifp;
1200         ifp->if_softc = vi;
1201
1202         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1203         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1204
1205         ifp->if_init = cxgbe_init;
1206         ifp->if_ioctl = cxgbe_ioctl;
1207         ifp->if_transmit = cxgbe_transmit;
1208         ifp->if_qflush = cxgbe_qflush;
1209         ifp->if_get_counter = cxgbe_get_counter;
1210
1211         ifp->if_capabilities = T4_CAP;
1212 #ifdef TCP_OFFLOAD
1213         if (vi->nofldrxq != 0)
1214                 ifp->if_capabilities |= IFCAP_TOE;
1215 #endif
1216         ifp->if_capenable = T4_CAP_ENABLE;
1217         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1218             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1219
1220         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1221         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1222         ifp->if_hw_tsomaxsegsize = 65536;
1223
1224         /* Initialize ifmedia for this VI */
1225         ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1226             cxgbe_media_status);
1227         build_medialist(vi->pi, &vi->media);
1228
1229         vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1230             EVENTHANDLER_PRI_ANY);
1231
1232         ether_ifattach(ifp, vi->hw_addr);
1233
1234         sb = sbuf_new_auto();
1235         sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1236 #ifdef TCP_OFFLOAD
1237         if (ifp->if_capabilities & IFCAP_TOE)
1238                 sbuf_printf(sb, "; %d txq, %d rxq (TOE)",
1239                     vi->nofldtxq, vi->nofldrxq);
1240 #endif
1241         sbuf_finish(sb);
1242         device_printf(dev, "%s\n", sbuf_data(sb));
1243         sbuf_delete(sb);
1244
1245         vi_sysctls(vi);
1246
1247         return (0);
1248 }
1249
1250 static int
1251 cxgbe_attach(device_t dev)
1252 {
1253         struct port_info *pi = device_get_softc(dev);
1254         struct vi_info *vi;
1255         int i, rc;
1256
1257         callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1258
1259         rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1260         if (rc)
1261                 return (rc);
1262
1263         for_each_vi(pi, i, vi) {
1264                 if (i == 0)
1265                         continue;
1266 #ifdef DEV_NETMAP
1267                 if (vi->flags & VI_NETMAP) {
1268                         /*
1269                          * media handled here to keep
1270                          * implementation private to this file
1271                          */
1272                         ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1273                             cxgbe_media_status);
1274                         build_medialist(pi, &vi->media);
1275                         vi->dev = device_add_child(dev, is_t4(pi->adapter) ?
1276                             "ncxgbe" : "ncxl", device_get_unit(dev));
1277                 } else
1278 #endif
1279                         vi->dev = device_add_child(dev, is_t4(pi->adapter) ?
1280                             "vcxgbe" : "vcxl", -1);
1281                 if (vi->dev == NULL) {
1282                         device_printf(dev, "failed to add VI %d\n", i);
1283                         continue;
1284                 }
1285                 device_set_softc(vi->dev, vi);
1286         }
1287
1288         cxgbe_sysctls(pi);
1289
1290         bus_generic_attach(dev);
1291
1292         return (0);
1293 }
1294
1295 static void
1296 cxgbe_vi_detach(struct vi_info *vi)
1297 {
1298         struct ifnet *ifp = vi->ifp;
1299
1300         ether_ifdetach(ifp);
1301
1302         if (vi->vlan_c)
1303                 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c);
1304
1305         /* Let detach proceed even if these fail. */
1306         cxgbe_uninit_synchronized(vi);
1307         callout_drain(&vi->tick);
1308         vi_full_uninit(vi);
1309
1310         ifmedia_removeall(&vi->media);
1311         if_free(vi->ifp);
1312         vi->ifp = NULL;
1313 }
1314
1315 static int
1316 cxgbe_detach(device_t dev)
1317 {
1318         struct port_info *pi = device_get_softc(dev);
1319         struct adapter *sc = pi->adapter;
1320         int rc;
1321
1322         /* Detach the extra VIs first. */
1323         rc = bus_generic_detach(dev);
1324         if (rc)
1325                 return (rc);
1326         device_delete_children(dev);
1327
1328         doom_vi(sc, &pi->vi[0]);
1329
1330         if (pi->flags & HAS_TRACEQ) {
1331                 sc->traceq = -1;        /* cloner should not create ifnet */
1332                 t4_tracer_port_detach(sc);
1333         }
1334
1335         cxgbe_vi_detach(&pi->vi[0]);
1336         callout_drain(&pi->tick);
1337
1338         end_synchronized_op(sc, 0);
1339
1340         return (0);
1341 }
1342
1343 static void
1344 cxgbe_init(void *arg)
1345 {
1346         struct vi_info *vi = arg;
1347         struct adapter *sc = vi->pi->adapter;
1348
1349         if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1350                 return;
1351         cxgbe_init_synchronized(vi);
1352         end_synchronized_op(sc, 0);
1353 }
1354
1355 static int
1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1357 {
1358         int rc = 0, mtu, flags, can_sleep;
1359         struct vi_info *vi = ifp->if_softc;
1360         struct adapter *sc = vi->pi->adapter;
1361         struct ifreq *ifr = (struct ifreq *)data;
1362         uint32_t mask;
1363
1364         switch (cmd) {
1365         case SIOCSIFMTU:
1366                 mtu = ifr->ifr_mtu;
1367                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1368                         return (EINVAL);
1369
1370                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1371                 if (rc)
1372                         return (rc);
1373                 ifp->if_mtu = mtu;
1374                 if (vi->flags & VI_INIT_DONE) {
1375                         t4_update_fl_bufsize(ifp);
1376                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1377                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1378                 }
1379                 end_synchronized_op(sc, 0);
1380                 break;
1381
1382         case SIOCSIFFLAGS:
1383                 can_sleep = 0;
1384 redo_sifflags:
1385                 rc = begin_synchronized_op(sc, vi,
1386                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1387                 if (rc)
1388                         return (rc);
1389
1390                 if (ifp->if_flags & IFF_UP) {
1391                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1392                                 flags = vi->if_flags;
1393                                 if ((ifp->if_flags ^ flags) &
1394                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1395                                         if (can_sleep == 1) {
1396                                                 end_synchronized_op(sc, 0);
1397                                                 can_sleep = 0;
1398                                                 goto redo_sifflags;
1399                                         }
1400                                         rc = update_mac_settings(ifp,
1401                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1402                                 }
1403                         } else {
1404                                 if (can_sleep == 0) {
1405                                         end_synchronized_op(sc, LOCK_HELD);
1406                                         can_sleep = 1;
1407                                         goto redo_sifflags;
1408                                 }
1409                                 rc = cxgbe_init_synchronized(vi);
1410                         }
1411                         vi->if_flags = ifp->if_flags;
1412                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1413                         if (can_sleep == 0) {
1414                                 end_synchronized_op(sc, LOCK_HELD);
1415                                 can_sleep = 1;
1416                                 goto redo_sifflags;
1417                         }
1418                         rc = cxgbe_uninit_synchronized(vi);
1419                 }
1420                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1421                 break;
1422
1423         case SIOCADDMULTI:
1424         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1425                 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi");
1426                 if (rc)
1427                         return (rc);
1428                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1429                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1430                 end_synchronized_op(sc, LOCK_HELD);
1431                 break;
1432
1433         case SIOCSIFCAP:
1434                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1435                 if (rc)
1436                         return (rc);
1437
1438                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1439                 if (mask & IFCAP_TXCSUM) {
1440                         ifp->if_capenable ^= IFCAP_TXCSUM;
1441                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1442
1443                         if (IFCAP_TSO4 & ifp->if_capenable &&
1444                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1445                                 ifp->if_capenable &= ~IFCAP_TSO4;
1446                                 if_printf(ifp,
1447                                     "tso4 disabled due to -txcsum.\n");
1448                         }
1449                 }
1450                 if (mask & IFCAP_TXCSUM_IPV6) {
1451                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1452                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1453
1454                         if (IFCAP_TSO6 & ifp->if_capenable &&
1455                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1456                                 ifp->if_capenable &= ~IFCAP_TSO6;
1457                                 if_printf(ifp,
1458                                     "tso6 disabled due to -txcsum6.\n");
1459                         }
1460                 }
1461                 if (mask & IFCAP_RXCSUM)
1462                         ifp->if_capenable ^= IFCAP_RXCSUM;
1463                 if (mask & IFCAP_RXCSUM_IPV6)
1464                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1465
1466                 /*
1467                  * Note that we leave CSUM_TSO alone (it is always set).  The
1468                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1469                  * sending a TSO request our way, so it's sufficient to toggle
1470                  * IFCAP_TSOx only.
1471                  */
1472                 if (mask & IFCAP_TSO4) {
1473                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1474                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1475                                 if_printf(ifp, "enable txcsum first.\n");
1476                                 rc = EAGAIN;
1477                                 goto fail;
1478                         }
1479                         ifp->if_capenable ^= IFCAP_TSO4;
1480                 }
1481                 if (mask & IFCAP_TSO6) {
1482                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1483                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1484                                 if_printf(ifp, "enable txcsum6 first.\n");
1485                                 rc = EAGAIN;
1486                                 goto fail;
1487                         }
1488                         ifp->if_capenable ^= IFCAP_TSO6;
1489                 }
1490                 if (mask & IFCAP_LRO) {
1491 #if defined(INET) || defined(INET6)
1492                         int i;
1493                         struct sge_rxq *rxq;
1494
1495                         ifp->if_capenable ^= IFCAP_LRO;
1496                         for_each_rxq(vi, i, rxq) {
1497                                 if (ifp->if_capenable & IFCAP_LRO)
1498                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1499                                 else
1500                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1501                         }
1502 #endif
1503                 }
1504 #ifdef TCP_OFFLOAD
1505                 if (mask & IFCAP_TOE) {
1506                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1507
1508                         rc = toe_capability(vi, enable);
1509                         if (rc != 0)
1510                                 goto fail;
1511
1512                         ifp->if_capenable ^= mask;
1513                 }
1514 #endif
1515                 if (mask & IFCAP_VLAN_HWTAGGING) {
1516                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1517                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1518                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1519                 }
1520                 if (mask & IFCAP_VLAN_MTU) {
1521                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1522
1523                         /* Need to find out how to disable auto-mtu-inflation */
1524                 }
1525                 if (mask & IFCAP_VLAN_HWTSO)
1526                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1527                 if (mask & IFCAP_VLAN_HWCSUM)
1528                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1529
1530 #ifdef VLAN_CAPABILITIES
1531                 VLAN_CAPABILITIES(ifp);
1532 #endif
1533 fail:
1534                 end_synchronized_op(sc, 0);
1535                 break;
1536
1537         case SIOCSIFMEDIA:
1538         case SIOCGIFMEDIA:
1539                 ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
1540                 break;
1541
1542         case SIOCGI2C: {
1543                 struct ifi2creq i2c;
1544
1545                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1546                 if (rc != 0)
1547                         break;
1548                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1549                         rc = EPERM;
1550                         break;
1551                 }
1552                 if (i2c.len > sizeof(i2c.data)) {
1553                         rc = EINVAL;
1554                         break;
1555                 }
1556                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1557                 if (rc)
1558                         return (rc);
1559                 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr,
1560                     i2c.offset, i2c.len, &i2c.data[0]);
1561                 end_synchronized_op(sc, 0);
1562                 if (rc == 0)
1563                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1564                 break;
1565         }
1566
1567         default:
1568                 rc = ether_ioctl(ifp, cmd, data);
1569         }
1570
1571         return (rc);
1572 }
1573
1574 static int
1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1576 {
1577         struct vi_info *vi = ifp->if_softc;
1578         struct port_info *pi = vi->pi;
1579         struct adapter *sc = pi->adapter;
1580         struct sge_txq *txq;
1581         void *items[1];
1582         int rc;
1583
1584         M_ASSERTPKTHDR(m);
1585         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1586
1587         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1588                 m_freem(m);
1589                 return (ENETDOWN);
1590         }
1591
1592         rc = parse_pkt(&m);
1593         if (__predict_false(rc != 0)) {
1594                 MPASS(m == NULL);                       /* was freed already */
1595                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1596                 return (rc);
1597         }
1598
1599         /* Select a txq. */
1600         txq = &sc->sge.txq[vi->first_txq];
1601         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1602                 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1603                     vi->rsrv_noflowq);
1604
1605         items[0] = m;
1606         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1607         if (__predict_false(rc != 0))
1608                 m_freem(m);
1609
1610         return (rc);
1611 }
1612
1613 static void
1614 cxgbe_qflush(struct ifnet *ifp)
1615 {
1616         struct vi_info *vi = ifp->if_softc;
1617         struct sge_txq *txq;
1618         int i;
1619
1620         /* queues do not exist if !VI_INIT_DONE. */
1621         if (vi->flags & VI_INIT_DONE) {
1622                 for_each_txq(vi, i, txq) {
1623                         TXQ_LOCK(txq);
1624                         txq->eq.flags &= ~EQ_ENABLED;
1625                         TXQ_UNLOCK(txq);
1626                         while (!mp_ring_is_idle(txq->r)) {
1627                                 mp_ring_check_drainage(txq->r, 0);
1628                                 pause("qflush", 1);
1629                         }
1630                 }
1631         }
1632         if_qflush(ifp);
1633 }
1634
1635 static uint64_t
1636 vi_get_counter(struct ifnet *ifp, ift_counter c)
1637 {
1638         struct vi_info *vi = ifp->if_softc;
1639         struct fw_vi_stats_vf *s = &vi->stats;
1640
1641         vi_refresh_stats(vi->pi->adapter, vi);
1642
1643         switch (c) {
1644         case IFCOUNTER_IPACKETS:
1645                 return (s->rx_bcast_frames + s->rx_mcast_frames +
1646                     s->rx_ucast_frames);
1647         case IFCOUNTER_IERRORS:
1648                 return (s->rx_err_frames);
1649         case IFCOUNTER_OPACKETS:
1650                 return (s->tx_bcast_frames + s->tx_mcast_frames +
1651                     s->tx_ucast_frames + s->tx_offload_frames);
1652         case IFCOUNTER_OERRORS:
1653                 return (s->tx_drop_frames);
1654         case IFCOUNTER_IBYTES:
1655                 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
1656                     s->rx_ucast_bytes);
1657         case IFCOUNTER_OBYTES:
1658                 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
1659                     s->tx_ucast_bytes + s->tx_offload_bytes);
1660         case IFCOUNTER_IMCASTS:
1661                 return (s->rx_mcast_frames);
1662         case IFCOUNTER_OMCASTS:
1663                 return (s->tx_mcast_frames);
1664         case IFCOUNTER_OQDROPS: {
1665                 uint64_t drops;
1666
1667                 drops = 0;
1668                 if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) {
1669                         int i;
1670                         struct sge_txq *txq;
1671
1672                         for_each_txq(vi, i, txq)
1673                                 drops += counter_u64_fetch(txq->r->drops);
1674                 }
1675
1676                 return (drops);
1677
1678         }
1679
1680         default:
1681                 return (if_get_counter_default(ifp, c));
1682         }
1683 }
1684
1685 uint64_t
1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1687 {
1688         struct vi_info *vi = ifp->if_softc;
1689         struct port_info *pi = vi->pi;
1690         struct adapter *sc = pi->adapter;
1691         struct port_stats *s = &pi->stats;
1692
1693         if (pi->nvi > 1)
1694                 return (vi_get_counter(ifp, c));
1695
1696         cxgbe_refresh_stats(sc, pi);
1697
1698         switch (c) {
1699         case IFCOUNTER_IPACKETS:
1700                 return (s->rx_frames - s->rx_pause);
1701
1702         case IFCOUNTER_IERRORS:
1703                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1704                     s->rx_fcs_err + s->rx_len_err);
1705
1706         case IFCOUNTER_OPACKETS:
1707                 return (s->tx_frames - s->tx_pause);
1708
1709         case IFCOUNTER_OERRORS:
1710                 return (s->tx_error_frames);
1711
1712         case IFCOUNTER_IBYTES:
1713                 return (s->rx_octets - s->rx_pause * 64);
1714
1715         case IFCOUNTER_OBYTES:
1716                 return (s->tx_octets - s->tx_pause * 64);
1717
1718         case IFCOUNTER_IMCASTS:
1719                 return (s->rx_mcast_frames - s->rx_pause);
1720
1721         case IFCOUNTER_OMCASTS:
1722                 return (s->tx_mcast_frames - s->tx_pause);
1723
1724         case IFCOUNTER_IQDROPS:
1725                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1726                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1727                     s->rx_trunc3 + pi->tnl_cong_drops);
1728
1729         case IFCOUNTER_OQDROPS: {
1730                 uint64_t drops;
1731
1732                 drops = s->tx_drop;
1733                 if (vi->flags & VI_INIT_DONE) {
1734                         int i;
1735                         struct sge_txq *txq;
1736
1737                         for_each_txq(vi, i, txq)
1738                                 drops += counter_u64_fetch(txq->r->drops);
1739                 }
1740
1741                 return (drops);
1742
1743         }
1744
1745         default:
1746                 return (if_get_counter_default(ifp, c));
1747         }
1748 }
1749
1750 static int
1751 cxgbe_media_change(struct ifnet *ifp)
1752 {
1753         struct vi_info *vi = ifp->if_softc;
1754
1755         device_printf(vi->dev, "%s unimplemented.\n", __func__);
1756
1757         return (EOPNOTSUPP);
1758 }
1759
1760 static void
1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1762 {
1763         struct vi_info *vi = ifp->if_softc;
1764         struct port_info *pi = vi->pi;
1765         struct ifmedia_entry *cur;
1766         int speed = pi->link_cfg.speed;
1767
1768         cur = vi->media.ifm_cur;
1769
1770         ifmr->ifm_status = IFM_AVALID;
1771         if (!pi->link_cfg.link_ok)
1772                 return;
1773
1774         ifmr->ifm_status |= IFM_ACTIVE;
1775
1776         /* active and current will differ iff current media is autoselect. */
1777         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1778                 return;
1779
1780         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1781         if (speed == SPEED_10000)
1782                 ifmr->ifm_active |= IFM_10G_T;
1783         else if (speed == SPEED_1000)
1784                 ifmr->ifm_active |= IFM_1000_T;
1785         else if (speed == SPEED_100)
1786                 ifmr->ifm_active |= IFM_100_TX;
1787         else if (speed == SPEED_10)
1788                 ifmr->ifm_active |= IFM_10_T;
1789         else
1790                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1791                             speed));
1792 }
1793
1794 static int
1795 vcxgbe_probe(device_t dev)
1796 {
1797         char buf[128];
1798         struct vi_info *vi = device_get_softc(dev);
1799
1800         snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
1801             vi - vi->pi->vi);
1802         device_set_desc_copy(dev, buf);
1803
1804         return (BUS_PROBE_DEFAULT);
1805 }
1806
1807 static int
1808 vcxgbe_attach(device_t dev)
1809 {
1810         struct vi_info *vi;
1811         struct port_info *pi;
1812         struct adapter *sc;
1813         int func, index, rc;
1814         u32 param, val;
1815
1816         vi = device_get_softc(dev);
1817         pi = vi->pi;
1818         sc = pi->adapter;
1819
1820         index = vi - pi->vi;
1821         KASSERT(index < nitems(vi_mac_funcs),
1822             ("%s: VI %s doesn't have a MAC func", __func__,
1823             device_get_nameunit(dev)));
1824         func = vi_mac_funcs[index];
1825         rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
1826             vi->hw_addr, &vi->rss_size, func, 0);
1827         if (rc < 0) {
1828                 device_printf(dev, "Failed to allocate virtual interface "
1829                     "for port %d: %d\n", pi->port_id, -rc);
1830                 return (-rc);
1831         }
1832         vi->viid = rc;
1833
1834         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
1835             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
1836             V_FW_PARAMS_PARAM_YZ(vi->viid);
1837         rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1838         if (rc)
1839                 vi->rss_base = 0xffff;
1840         else {
1841                 /* MPASS((val >> 16) == rss_size); */
1842                 vi->rss_base = val & 0xffff;
1843         }
1844
1845         rc = cxgbe_vi_attach(dev, vi);
1846         if (rc) {
1847                 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1848                 return (rc);
1849         }
1850         return (0);
1851 }
1852
1853 static int
1854 vcxgbe_detach(device_t dev)
1855 {
1856         struct vi_info *vi;
1857         struct adapter *sc;
1858
1859         vi = device_get_softc(dev);
1860         sc = vi->pi->adapter;
1861
1862         doom_vi(sc, vi);
1863
1864         cxgbe_vi_detach(vi);
1865         t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1866
1867         end_synchronized_op(sc, 0);
1868
1869         return (0);
1870 }
1871
1872 void
1873 t4_fatal_err(struct adapter *sc)
1874 {
1875         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1876         t4_intr_disable(sc);
1877         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1878             device_get_nameunit(sc->dev));
1879 }
1880
1881 static int
1882 map_bars_0_and_4(struct adapter *sc)
1883 {
1884         sc->regs_rid = PCIR_BAR(0);
1885         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1886             &sc->regs_rid, RF_ACTIVE);
1887         if (sc->regs_res == NULL) {
1888                 device_printf(sc->dev, "cannot map registers.\n");
1889                 return (ENXIO);
1890         }
1891         sc->bt = rman_get_bustag(sc->regs_res);
1892         sc->bh = rman_get_bushandle(sc->regs_res);
1893         sc->mmio_len = rman_get_size(sc->regs_res);
1894         setbit(&sc->doorbells, DOORBELL_KDB);
1895
1896         sc->msix_rid = PCIR_BAR(4);
1897         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1898             &sc->msix_rid, RF_ACTIVE);
1899         if (sc->msix_res == NULL) {
1900                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1901                 return (ENXIO);
1902         }
1903
1904         return (0);
1905 }
1906
1907 static int
1908 map_bar_2(struct adapter *sc)
1909 {
1910
1911         /*
1912          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1913          * to map it if RDMA is disabled.
1914          */
1915         if (is_t4(sc) && sc->rdmacaps == 0)
1916                 return (0);
1917
1918         sc->udbs_rid = PCIR_BAR(2);
1919         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1920             &sc->udbs_rid, RF_ACTIVE);
1921         if (sc->udbs_res == NULL) {
1922                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1923                 return (ENXIO);
1924         }
1925         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1926
1927         if (is_t5(sc)) {
1928                 setbit(&sc->doorbells, DOORBELL_UDB);
1929 #if defined(__i386__) || defined(__amd64__)
1930                 if (t5_write_combine) {
1931                         int rc;
1932
1933                         /*
1934                          * Enable write combining on BAR2.  This is the
1935                          * userspace doorbell BAR and is split into 128B
1936                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1937                          * with an egress queue.  The first 64B has the doorbell
1938                          * and the second 64B can be used to submit a tx work
1939                          * request with an implicit doorbell.
1940                          */
1941
1942                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1943                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1944                         if (rc == 0) {
1945                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1946                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1947                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1948                         } else {
1949                                 device_printf(sc->dev,
1950                                     "couldn't enable write combining: %d\n",
1951                                     rc);
1952                         }
1953
1954                         t4_write_reg(sc, A_SGE_STAT_CFG,
1955                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1956                 }
1957 #endif
1958         }
1959
1960         return (0);
1961 }
1962
1963 static const struct memwin t4_memwin[] = {
1964         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1965         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1966         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1967 };
1968
1969 static const struct memwin t5_memwin[] = {
1970         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1971         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1972         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1973 };
1974
1975 static void
1976 setup_memwin(struct adapter *sc)
1977 {
1978         const struct memwin *mw;
1979         int i, n;
1980         uint32_t bar0;
1981
1982         if (is_t4(sc)) {
1983                 /*
1984                  * Read low 32b of bar0 indirectly via the hardware backdoor
1985                  * mechanism.  Works from within PCI passthrough environments
1986                  * too, where rman_get_start() can return a different value.  We
1987                  * need to program the T4 memory window decoders with the actual
1988                  * addresses that will be coming across the PCIe link.
1989                  */
1990                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1991                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1992
1993                 mw = &t4_memwin[0];
1994                 n = nitems(t4_memwin);
1995         } else {
1996                 /* T5 uses the relative offset inside the PCIe BAR */
1997                 bar0 = 0;
1998
1999                 mw = &t5_memwin[0];
2000                 n = nitems(t5_memwin);
2001         }
2002
2003         for (i = 0; i < n; i++, mw++) {
2004                 t4_write_reg(sc,
2005                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
2006                     (mw->base + bar0) | V_BIR(0) |
2007                     V_WINDOW(ilog2(mw->aperture) - 10));
2008         }
2009
2010         /* flush */
2011         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
2012 }
2013
2014 /*
2015  * Verify that the memory range specified by the addr/len pair is valid and lies
2016  * entirely within a single region (EDCx or MCx).
2017  */
2018 static int
2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
2020 {
2021         uint32_t em, addr_len, maddr, mlen;
2022
2023         /* Memory can only be accessed in naturally aligned 4 byte units */
2024         if (addr & 3 || len & 3 || len == 0)
2025                 return (EINVAL);
2026
2027         /* Enabled memories */
2028         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2029         if (em & F_EDRAM0_ENABLE) {
2030                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2031                 maddr = G_EDRAM0_BASE(addr_len) << 20;
2032                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
2033                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2034                     addr + len <= maddr + mlen)
2035                         return (0);
2036         }
2037         if (em & F_EDRAM1_ENABLE) {
2038                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2039                 maddr = G_EDRAM1_BASE(addr_len) << 20;
2040                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
2041                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2042                     addr + len <= maddr + mlen)
2043                         return (0);
2044         }
2045         if (em & F_EXT_MEM_ENABLE) {
2046                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2047                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
2048                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
2049                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2050                     addr + len <= maddr + mlen)
2051                         return (0);
2052         }
2053         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
2054                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2055                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2056                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
2057                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2058                     addr + len <= maddr + mlen)
2059                         return (0);
2060         }
2061
2062         return (EFAULT);
2063 }
2064
2065 static int
2066 fwmtype_to_hwmtype(int mtype)
2067 {
2068
2069         switch (mtype) {
2070         case FW_MEMTYPE_EDC0:
2071                 return (MEM_EDC0);
2072         case FW_MEMTYPE_EDC1:
2073                 return (MEM_EDC1);
2074         case FW_MEMTYPE_EXTMEM:
2075                 return (MEM_MC0);
2076         case FW_MEMTYPE_EXTMEM1:
2077                 return (MEM_MC1);
2078         default:
2079                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
2080         }
2081 }
2082
2083 /*
2084  * Verify that the memory range specified by the memtype/offset/len pair is
2085  * valid and lies entirely within the memtype specified.  The global address of
2086  * the start of the range is returned in addr.
2087  */
2088 static int
2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
2090     uint32_t *addr)
2091 {
2092         uint32_t em, addr_len, maddr, mlen;
2093
2094         /* Memory can only be accessed in naturally aligned 4 byte units */
2095         if (off & 3 || len & 3 || len == 0)
2096                 return (EINVAL);
2097
2098         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2099         switch (fwmtype_to_hwmtype(mtype)) {
2100         case MEM_EDC0:
2101                 if (!(em & F_EDRAM0_ENABLE))
2102                         return (EINVAL);
2103                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2104                 maddr = G_EDRAM0_BASE(addr_len) << 20;
2105                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
2106                 break;
2107         case MEM_EDC1:
2108                 if (!(em & F_EDRAM1_ENABLE))
2109                         return (EINVAL);
2110                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2111                 maddr = G_EDRAM1_BASE(addr_len) << 20;
2112                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
2113                 break;
2114         case MEM_MC:
2115                 if (!(em & F_EXT_MEM_ENABLE))
2116                         return (EINVAL);
2117                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2118                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
2119                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
2120                 break;
2121         case MEM_MC1:
2122                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
2123                         return (EINVAL);
2124                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2125                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2126                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
2127                 break;
2128         default:
2129                 return (EINVAL);
2130         }
2131
2132         if (mlen > 0 && off < mlen && off + len <= mlen) {
2133                 *addr = maddr + off;    /* global address */
2134                 return (0);
2135         }
2136
2137         return (EFAULT);
2138 }
2139
2140 static void
2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
2142 {
2143         const struct memwin *mw;
2144
2145         if (is_t4(sc)) {
2146                 KASSERT(win >= 0 && win < nitems(t4_memwin),
2147                     ("%s: incorrect memwin# (%d)", __func__, win));
2148                 mw = &t4_memwin[win];
2149         } else {
2150                 KASSERT(win >= 0 && win < nitems(t5_memwin),
2151                     ("%s: incorrect memwin# (%d)", __func__, win));
2152                 mw = &t5_memwin[win];
2153         }
2154
2155         if (base != NULL)
2156                 *base = mw->base;
2157         if (aperture != NULL)
2158                 *aperture = mw->aperture;
2159 }
2160
2161 /*
2162  * Positions the memory window such that it can be used to access the specified
2163  * address in the chip's address space.  The return value is the offset of addr
2164  * from the start of the window.
2165  */
2166 static uint32_t
2167 position_memwin(struct adapter *sc, int n, uint32_t addr)
2168 {
2169         uint32_t start, pf;
2170         uint32_t reg;
2171
2172         KASSERT(n >= 0 && n <= 3,
2173             ("%s: invalid window %d.", __func__, n));
2174         KASSERT((addr & 3) == 0,
2175             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
2176
2177         if (is_t4(sc)) {
2178                 pf = 0;
2179                 start = addr & ~0xf;    /* start must be 16B aligned */
2180         } else {
2181                 pf = V_PFNUM(sc->pf);
2182                 start = addr & ~0x7f;   /* start must be 128B aligned */
2183         }
2184         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
2185
2186         t4_write_reg(sc, reg, start | pf);
2187         t4_read_reg(sc, reg);
2188
2189         return (addr - start);
2190 }
2191
2192 static int
2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis,
2194     struct intrs_and_queues *iaq)
2195 {
2196         int rc, itype, navail, nrxq10g, nrxq1g, n;
2197         int nofldrxq10g = 0, nofldrxq1g = 0;
2198         int nnmrxq10g = 0, nnmrxq1g = 0;
2199
2200         bzero(iaq, sizeof(*iaq));
2201
2202         iaq->ntxq10g = t4_ntxq10g;
2203         iaq->ntxq1g = t4_ntxq1g;
2204         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
2205         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
2206         iaq->rsrv_noflowq = t4_rsrv_noflowq;
2207 #ifdef TCP_OFFLOAD
2208         if (is_offload(sc)) {
2209                 iaq->nofldtxq10g = t4_nofldtxq10g;
2210                 iaq->nofldtxq1g = t4_nofldtxq1g;
2211                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
2212                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
2213         }
2214 #endif
2215 #ifdef DEV_NETMAP
2216         iaq->nnmtxq10g = t4_nnmtxq10g;
2217         iaq->nnmtxq1g = t4_nnmtxq1g;
2218         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
2219         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
2220 #endif
2221
2222         for (itype = INTR_MSIX; itype; itype >>= 1) {
2223
2224                 if ((itype & t4_intr_types) == 0)
2225                         continue;       /* not allowed */
2226
2227                 if (itype == INTR_MSIX)
2228                         navail = pci_msix_count(sc->dev);
2229                 else if (itype == INTR_MSI)
2230                         navail = pci_msi_count(sc->dev);
2231                 else
2232                         navail = 1;
2233 restart:
2234                 if (navail == 0)
2235                         continue;
2236
2237                 iaq->intr_type = itype;
2238                 iaq->intr_flags_10g = 0;
2239                 iaq->intr_flags_1g = 0;
2240
2241                 /*
2242                  * Best option: an interrupt vector for errors, one for the
2243                  * firmware event queue, and one for every rxq (NIC, TOE, and
2244                  * netmap).
2245                  */
2246                 iaq->nirq = T4_EXTRA_INTR;
2247                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
2248                 iaq->nirq += n10g * 2 * (num_vis - 1);
2249                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
2250                 iaq->nirq += n1g * 2 * (num_vis - 1);
2251                 if (iaq->nirq <= navail &&
2252                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2253                         iaq->intr_flags_10g = INTR_ALL;
2254                         iaq->intr_flags_1g = INTR_ALL;
2255                         goto allocate;
2256                 }
2257
2258                 /*
2259                  * Second best option: a vector for errors, one for the firmware
2260                  * event queue, and vectors for either all the NIC rx queues or
2261                  * all the TOE rx queues.  The queues that don't get vectors
2262                  * will forward their interrupts to those that do.
2263                  *
2264                  * Note: netmap rx queues cannot be created early and so they
2265                  * can't be setup to receive forwarded interrupts for others.
2266                  */
2267                 iaq->nirq = T4_EXTRA_INTR;
2268                 if (nrxq10g >= nofldrxq10g) {
2269                         iaq->intr_flags_10g = INTR_RXQ;
2270                         iaq->nirq += n10g * nrxq10g;
2271                         iaq->nirq += n10g * (num_vis - 1);
2272 #ifdef DEV_NETMAP
2273                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2274 #endif
2275                 } else {
2276                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2277                         iaq->nirq += n10g * nofldrxq10g;
2278 #ifdef DEV_NETMAP
2279                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2280 #endif
2281                 }
2282                 if (nrxq1g >= nofldrxq1g) {
2283                         iaq->intr_flags_1g = INTR_RXQ;
2284                         iaq->nirq += n1g * nrxq1g;
2285                         iaq->nirq += n1g * (num_vis - 1);
2286 #ifdef DEV_NETMAP
2287                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2288 #endif
2289                 } else {
2290                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2291                         iaq->nirq += n1g * nofldrxq1g;
2292 #ifdef DEV_NETMAP
2293                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2294 #endif
2295                 }
2296                 if (iaq->nirq <= navail &&
2297                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2298                         goto allocate;
2299
2300                 /*
2301                  * Next best option: an interrupt vector for errors, one for the
2302                  * firmware event queue, and at least one per VI.  At this
2303                  * point we know we'll have to downsize nrxq and/or nofldrxq
2304                  * and/or nnmrxq to fit what's available to us.
2305                  */
2306                 iaq->nirq = T4_EXTRA_INTR;
2307                 iaq->nirq += (n10g + n1g) * num_vis;
2308                 if (iaq->nirq <= navail) {
2309                         int leftover = navail - iaq->nirq;
2310
2311                         if (n10g > 0) {
2312                                 int target = max(nrxq10g, nofldrxq10g);
2313
2314                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2315                                     INTR_RXQ : INTR_OFLD_RXQ;
2316
2317                                 n = 1;
2318                                 while (n < target && leftover >= n10g) {
2319                                         leftover -= n10g;
2320                                         iaq->nirq += n10g;
2321                                         n++;
2322                                 }
2323                                 iaq->nrxq10g = min(n, nrxq10g);
2324 #ifdef TCP_OFFLOAD
2325                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2326 #endif
2327 #ifdef DEV_NETMAP
2328                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2329 #endif
2330                         }
2331
2332                         if (n1g > 0) {
2333                                 int target = max(nrxq1g, nofldrxq1g);
2334
2335                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2336                                     INTR_RXQ : INTR_OFLD_RXQ;
2337
2338                                 n = 1;
2339                                 while (n < target && leftover >= n1g) {
2340                                         leftover -= n1g;
2341                                         iaq->nirq += n1g;
2342                                         n++;
2343                                 }
2344                                 iaq->nrxq1g = min(n, nrxq1g);
2345 #ifdef TCP_OFFLOAD
2346                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2347 #endif
2348 #ifdef DEV_NETMAP
2349                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2350 #endif
2351                         }
2352
2353                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2354                                 goto allocate;
2355                 }
2356
2357                 /*
2358                  * Least desirable option: one interrupt vector for everything.
2359                  */
2360                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2361                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2362 #ifdef TCP_OFFLOAD
2363                 if (is_offload(sc))
2364                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2365 #endif
2366 #ifdef DEV_NETMAP
2367                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2368 #endif
2369
2370 allocate:
2371                 navail = iaq->nirq;
2372                 rc = 0;
2373                 if (itype == INTR_MSIX)
2374                         rc = pci_alloc_msix(sc->dev, &navail);
2375                 else if (itype == INTR_MSI)
2376                         rc = pci_alloc_msi(sc->dev, &navail);
2377
2378                 if (rc == 0) {
2379                         if (navail == iaq->nirq)
2380                                 return (0);
2381
2382                         /*
2383                          * Didn't get the number requested.  Use whatever number
2384                          * the kernel is willing to allocate (it's in navail).
2385                          */
2386                         device_printf(sc->dev, "fewer vectors than requested, "
2387                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2388                             itype, iaq->nirq, navail);
2389                         pci_release_msi(sc->dev);
2390                         goto restart;
2391                 }
2392
2393                 device_printf(sc->dev,
2394                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2395                     itype, rc, iaq->nirq, navail);
2396         }
2397
2398         device_printf(sc->dev,
2399             "failed to find a usable interrupt type.  "
2400             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2401             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2402
2403         return (ENXIO);
2404 }
2405
2406 #define FW_VERSION(chip) ( \
2407     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2408     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2409     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2410     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2412
2413 struct fw_info {
2414         uint8_t chip;
2415         char *kld_name;
2416         char *fw_mod_name;
2417         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2418 } fw_info[] = {
2419         {
2420                 .chip = CHELSIO_T4,
2421                 .kld_name = "t4fw_cfg",
2422                 .fw_mod_name = "t4fw",
2423                 .fw_hdr = {
2424                         .chip = FW_HDR_CHIP_T4,
2425                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2426                         .intfver_nic = FW_INTFVER(T4, NIC),
2427                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2428                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2429                         .intfver_ri = FW_INTFVER(T4, RI),
2430                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2431                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2432                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2433                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2434                 },
2435         }, {
2436                 .chip = CHELSIO_T5,
2437                 .kld_name = "t5fw_cfg",
2438                 .fw_mod_name = "t5fw",
2439                 .fw_hdr = {
2440                         .chip = FW_HDR_CHIP_T5,
2441                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2442                         .intfver_nic = FW_INTFVER(T5, NIC),
2443                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2444                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2445                         .intfver_ri = FW_INTFVER(T5, RI),
2446                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2447                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2448                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2449                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2450                 },
2451         }
2452 };
2453
2454 static struct fw_info *
2455 find_fw_info(int chip)
2456 {
2457         int i;
2458
2459         for (i = 0; i < nitems(fw_info); i++) {
2460                 if (fw_info[i].chip == chip)
2461                         return (&fw_info[i]);
2462         }
2463         return (NULL);
2464 }
2465
2466 /*
2467  * Is the given firmware API compatible with the one the driver was compiled
2468  * with?
2469  */
2470 static int
2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2472 {
2473
2474         /* short circuit if it's the exact same firmware version */
2475         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2476                 return (1);
2477
2478         /*
2479          * XXX: Is this too conservative?  Perhaps I should limit this to the
2480          * features that are supported in the driver.
2481          */
2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2483         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2484             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2485             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2486                 return (1);
2487 #undef SAME_INTF
2488
2489         return (0);
2490 }
2491
2492 /*
2493  * The firmware in the KLD is usable, but should it be installed?  This routine
2494  * explains itself in detail if it indicates the KLD firmware should be
2495  * installed.
2496  */
2497 static int
2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2499 {
2500         const char *reason;
2501
2502         if (!card_fw_usable) {
2503                 reason = "incompatible or unusable";
2504                 goto install;
2505         }
2506
2507         if (k > c) {
2508                 reason = "older than the version bundled with this driver";
2509                 goto install;
2510         }
2511
2512         if (t4_fw_install == 2 && k != c) {
2513                 reason = "different than the version bundled with this driver";
2514                 goto install;
2515         }
2516
2517         return (0);
2518
2519 install:
2520         if (t4_fw_install == 0) {
2521                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2522                     "but the driver is prohibited from installing a different "
2523                     "firmware on the card.\n",
2524                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2525                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2526
2527                 return (0);
2528         }
2529
2530         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2531             "installing firmware %u.%u.%u.%u on card.\n",
2532             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2533             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2534             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2535             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2536
2537         return (1);
2538 }
2539 /*
2540  * Establish contact with the firmware and determine if we are the master driver
2541  * or not, and whether we are responsible for chip initialization.
2542  */
2543 static int
2544 prep_firmware(struct adapter *sc)
2545 {
2546         const struct firmware *fw = NULL, *default_cfg;
2547         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2548         enum dev_state state;
2549         struct fw_info *fw_info;
2550         struct fw_hdr *card_fw;         /* fw on the card */
2551         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2552         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2553                                            against */
2554
2555         /* Contact firmware. */
2556         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2557         if (rc < 0 || state == DEV_STATE_ERR) {
2558                 rc = -rc;
2559                 device_printf(sc->dev,
2560                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2561                 return (rc);
2562         }
2563         pf = rc;
2564         if (pf == sc->mbox)
2565                 sc->flags |= MASTER_PF;
2566         else if (state == DEV_STATE_UNINIT) {
2567                 /*
2568                  * We didn't get to be the master so we definitely won't be
2569                  * configuring the chip.  It's a bug if someone else hasn't
2570                  * configured it already.
2571                  */
2572                 device_printf(sc->dev, "couldn't be master(%d), "
2573                     "device not already initialized either(%d).\n", rc, state);
2574                 return (EDOOFUS);
2575         }
2576
2577         /* This is the firmware whose headers the driver was compiled against */
2578         fw_info = find_fw_info(chip_id(sc));
2579         if (fw_info == NULL) {
2580                 device_printf(sc->dev,
2581                     "unable to look up firmware information for chip %d.\n",
2582                     chip_id(sc));
2583                 return (EINVAL);
2584         }
2585         drv_fw = &fw_info->fw_hdr;
2586
2587         /*
2588          * The firmware KLD contains many modules.  The KLD name is also the
2589          * name of the module that contains the default config file.
2590          */
2591         default_cfg = firmware_get(fw_info->kld_name);
2592
2593         /* Read the header of the firmware on the card */
2594         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2595         rc = -t4_read_flash(sc, FLASH_FW_START,
2596             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2597         if (rc == 0)
2598                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2599         else {
2600                 device_printf(sc->dev,
2601                     "Unable to read card's firmware header: %d\n", rc);
2602                 card_fw_usable = 0;
2603         }
2604
2605         /* This is the firmware in the KLD */
2606         fw = firmware_get(fw_info->fw_mod_name);
2607         if (fw != NULL) {
2608                 kld_fw = (const void *)fw->data;
2609                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2610         } else {
2611                 kld_fw = NULL;
2612                 kld_fw_usable = 0;
2613         }
2614
2615         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2616             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2617                 /*
2618                  * Common case: the firmware on the card is an exact match and
2619                  * the KLD is an exact match too, or the KLD is
2620                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2621                  * here -- use cxgbetool loadfw if you want to reinstall the
2622                  * same firmware as the one on the card.
2623                  */
2624         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2625             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2626             be32toh(card_fw->fw_ver))) {
2627
2628                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2629                 if (rc != 0) {
2630                         device_printf(sc->dev,
2631                             "failed to install firmware: %d\n", rc);
2632                         goto done;
2633                 }
2634
2635                 /* Installed successfully, update the cached header too. */
2636                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2637                 card_fw_usable = 1;
2638                 need_fw_reset = 0;      /* already reset as part of load_fw */
2639         }
2640
2641         if (!card_fw_usable) {
2642                 uint32_t d, c, k;
2643
2644                 d = ntohl(drv_fw->fw_ver);
2645                 c = ntohl(card_fw->fw_ver);
2646                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2647
2648                 device_printf(sc->dev, "Cannot find a usable firmware: "
2649                     "fw_install %d, chip state %d, "
2650                     "driver compiled with %d.%d.%d.%d, "
2651                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2652                     t4_fw_install, state,
2653                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2654                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2655                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2656                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2657                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2658                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2659                 rc = EINVAL;
2660                 goto done;
2661         }
2662
2663         /* We're using whatever's on the card and it's known to be good. */
2664         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2665         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2666             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2667             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2668             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2669             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2670         t4_get_tp_version(sc, &sc->params.tp_vers);
2671
2672         /* Reset device */
2673         if (need_fw_reset &&
2674             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2675                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2676                 if (rc != ETIMEDOUT && rc != EIO)
2677                         t4_fw_bye(sc, sc->mbox);
2678                 goto done;
2679         }
2680         sc->flags |= FW_OK;
2681
2682         rc = get_params__pre_init(sc);
2683         if (rc != 0)
2684                 goto done; /* error message displayed already */
2685
2686         /* Partition adapter resources as specified in the config file. */
2687         if (state == DEV_STATE_UNINIT) {
2688
2689                 KASSERT(sc->flags & MASTER_PF,
2690                     ("%s: trying to change chip settings when not master.",
2691                     __func__));
2692
2693                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2694                 if (rc != 0)
2695                         goto done;      /* error message displayed already */
2696
2697                 t4_tweak_chip_settings(sc);
2698
2699                 /* get basic stuff going */
2700                 rc = -t4_fw_initialize(sc, sc->mbox);
2701                 if (rc != 0) {
2702                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2703                         goto done;
2704                 }
2705         } else {
2706                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2707                 sc->cfcsum = 0;
2708         }
2709
2710 done:
2711         free(card_fw, M_CXGBE);
2712         if (fw != NULL)
2713                 firmware_put(fw, FIRMWARE_UNLOAD);
2714         if (default_cfg != NULL)
2715                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2716
2717         return (rc);
2718 }
2719
2720 #define FW_PARAM_DEV(param) \
2721         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2722          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2723 #define FW_PARAM_PFVF(param) \
2724         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2725          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2726
2727 /*
2728  * Partition chip resources for use between various PFs, VFs, etc.
2729  */
2730 static int
2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2732     const char *name_prefix)
2733 {
2734         const struct firmware *cfg = NULL;
2735         int rc = 0;
2736         struct fw_caps_config_cmd caps;
2737         uint32_t mtype, moff, finicsum, cfcsum;
2738
2739         /*
2740          * Figure out what configuration file to use.  Pick the default config
2741          * file for the card if the user hasn't specified one explicitly.
2742          */
2743         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2744         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2745                 /* Card specific overrides go here. */
2746                 if (pci_get_device(sc->dev) == 0x440a)
2747                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2748                 if (is_fpga(sc))
2749                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2750         }
2751
2752         /*
2753          * We need to load another module if the profile is anything except
2754          * "default" or "flash".
2755          */
2756         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2757             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2758                 char s[32];
2759
2760                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2761                 cfg = firmware_get(s);
2762                 if (cfg == NULL) {
2763                         if (default_cfg != NULL) {
2764                                 device_printf(sc->dev,
2765                                     "unable to load module \"%s\" for "
2766                                     "configuration profile \"%s\", will use "
2767                                     "the default config file instead.\n",
2768                                     s, sc->cfg_file);
2769                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2770                                     "%s", DEFAULT_CF);
2771                         } else {
2772                                 device_printf(sc->dev,
2773                                     "unable to load module \"%s\" for "
2774                                     "configuration profile \"%s\", will use "
2775                                     "the config file on the card's flash "
2776                                     "instead.\n", s, sc->cfg_file);
2777                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2778                                     "%s", FLASH_CF);
2779                         }
2780                 }
2781         }
2782
2783         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2784             default_cfg == NULL) {
2785                 device_printf(sc->dev,
2786                     "default config file not available, will use the config "
2787                     "file on the card's flash instead.\n");
2788                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2789         }
2790
2791         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2792                 u_int cflen, i, n;
2793                 const uint32_t *cfdata;
2794                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2795
2796                 KASSERT(cfg != NULL || default_cfg != NULL,
2797                     ("%s: no config to upload", __func__));
2798
2799                 /*
2800                  * Ask the firmware where it wants us to upload the config file.
2801                  */
2802                 param = FW_PARAM_DEV(CF);
2803                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2804                 if (rc != 0) {
2805                         /* No support for config file?  Shouldn't happen. */
2806                         device_printf(sc->dev,
2807                             "failed to query config file location: %d.\n", rc);
2808                         goto done;
2809                 }
2810                 mtype = G_FW_PARAMS_PARAM_Y(val);
2811                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2812
2813                 /*
2814                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2815                  * useless stuffing/comments at the end of the config file so
2816                  * it's ok to simply throw away the last remaining bytes when
2817                  * the config file is not an exact multiple of 4.  This also
2818                  * helps with the validate_mt_off_len check.
2819                  */
2820                 if (cfg != NULL) {
2821                         cflen = cfg->datasize & ~3;
2822                         cfdata = cfg->data;
2823                 } else {
2824                         cflen = default_cfg->datasize & ~3;
2825                         cfdata = default_cfg->data;
2826                 }
2827
2828                 if (cflen > FLASH_CFG_MAX_SIZE) {
2829                         device_printf(sc->dev,
2830                             "config file too long (%d, max allowed is %d).  "
2831                             "Will try to use the config on the card, if any.\n",
2832                             cflen, FLASH_CFG_MAX_SIZE);
2833                         goto use_config_on_flash;
2834                 }
2835
2836                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2837                 if (rc != 0) {
2838                         device_printf(sc->dev,
2839                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2840                             "Will try to use the config on the card, if any.\n",
2841                             __func__, mtype, moff, cflen, rc);
2842                         goto use_config_on_flash;
2843                 }
2844
2845                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2846                 while (cflen) {
2847                         off = position_memwin(sc, 2, addr);
2848                         n = min(cflen, mw_aperture - off);
2849                         for (i = 0; i < n; i += 4)
2850                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2851                         cflen -= n;
2852                         addr += n;
2853                 }
2854         } else {
2855 use_config_on_flash:
2856                 mtype = FW_MEMTYPE_FLASH;
2857                 moff = t4_flash_cfg_addr(sc);
2858         }
2859
2860         bzero(&caps, sizeof(caps));
2861         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2862             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2863         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2864             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2865             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2866         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2867         if (rc != 0) {
2868                 device_printf(sc->dev,
2869                     "failed to pre-process config file: %d "
2870                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2871                 goto done;
2872         }
2873
2874         finicsum = be32toh(caps.finicsum);
2875         cfcsum = be32toh(caps.cfcsum);
2876         if (finicsum != cfcsum) {
2877                 device_printf(sc->dev,
2878                     "WARNING: config file checksum mismatch: %08x %08x\n",
2879                     finicsum, cfcsum);
2880         }
2881         sc->cfcsum = cfcsum;
2882
2883 #define LIMIT_CAPS(x) do { \
2884         caps.x &= htobe16(t4_##x##_allowed); \
2885 } while (0)
2886
2887         /*
2888          * Let the firmware know what features will (not) be used so it can tune
2889          * things accordingly.
2890          */
2891         LIMIT_CAPS(linkcaps);
2892         LIMIT_CAPS(niccaps);
2893         LIMIT_CAPS(toecaps);
2894         LIMIT_CAPS(rdmacaps);
2895         LIMIT_CAPS(iscsicaps);
2896         LIMIT_CAPS(fcoecaps);
2897 #undef LIMIT_CAPS
2898
2899         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2900             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2901         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2902         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2903         if (rc != 0) {
2904                 device_printf(sc->dev,
2905                     "failed to process config file: %d.\n", rc);
2906         }
2907 done:
2908         if (cfg != NULL)
2909                 firmware_put(cfg, FIRMWARE_UNLOAD);
2910         return (rc);
2911 }
2912
2913 /*
2914  * Retrieve parameters that are needed (or nice to have) very early.
2915  */
2916 static int
2917 get_params__pre_init(struct adapter *sc)
2918 {
2919         int rc;
2920         uint32_t param[2], val[2];
2921         struct fw_devlog_cmd cmd;
2922         struct devlog_params *dlog = &sc->params.devlog;
2923
2924         param[0] = FW_PARAM_DEV(PORTVEC);
2925         param[1] = FW_PARAM_DEV(CCLK);
2926         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2927         if (rc != 0) {
2928                 device_printf(sc->dev,
2929                     "failed to query parameters (pre_init): %d.\n", rc);
2930                 return (rc);
2931         }
2932
2933         sc->params.portvec = val[0];
2934         sc->params.nports = bitcount32(val[0]);
2935         sc->params.vpd.cclk = val[1];
2936
2937         /* Read device log parameters. */
2938         bzero(&cmd, sizeof(cmd));
2939         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2940             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2941         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2942         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2943         if (rc != 0) {
2944                 device_printf(sc->dev,
2945                     "failed to get devlog parameters: %d.\n", rc);
2946                 bzero(dlog, sizeof (*dlog));
2947                 rc = 0; /* devlog isn't critical for device operation */
2948         } else {
2949                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2950                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2951                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2952                 dlog->size = be32toh(cmd.memsize_devlog);
2953         }
2954
2955         return (rc);
2956 }
2957
2958 /*
2959  * Retrieve various parameters that are of interest to the driver.  The device
2960  * has been initialized by the firmware at this point.
2961  */
2962 static int
2963 get_params__post_init(struct adapter *sc)
2964 {
2965         int rc;
2966         uint32_t param[7], val[7];
2967         struct fw_caps_config_cmd caps;
2968
2969         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2970         param[1] = FW_PARAM_PFVF(EQ_START);
2971         param[2] = FW_PARAM_PFVF(FILTER_START);
2972         param[3] = FW_PARAM_PFVF(FILTER_END);
2973         param[4] = FW_PARAM_PFVF(L2T_START);
2974         param[5] = FW_PARAM_PFVF(L2T_END);
2975         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2976         if (rc != 0) {
2977                 device_printf(sc->dev,
2978                     "failed to query parameters (post_init): %d.\n", rc);
2979                 return (rc);
2980         }
2981
2982         sc->sge.iq_start = val[0];
2983         sc->sge.eq_start = val[1];
2984         sc->tids.ftid_base = val[2];
2985         sc->tids.nftids = val[3] - val[2] + 1;
2986         sc->params.ftid_min = val[2];
2987         sc->params.ftid_max = val[3];
2988         sc->vres.l2t.start = val[4];
2989         sc->vres.l2t.size = val[5] - val[4] + 1;
2990         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2991             ("%s: L2 table size (%u) larger than expected (%u)",
2992             __func__, sc->vres.l2t.size, L2T_SIZE));
2993
2994         /* get capabilites */
2995         bzero(&caps, sizeof(caps));
2996         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2997             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2998         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2999         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3000         if (rc != 0) {
3001                 device_printf(sc->dev,
3002                     "failed to get card capabilities: %d.\n", rc);
3003                 return (rc);
3004         }
3005
3006 #define READ_CAPS(x) do { \
3007         sc->x = htobe16(caps.x); \
3008 } while (0)
3009         READ_CAPS(linkcaps);
3010         READ_CAPS(niccaps);
3011         READ_CAPS(toecaps);
3012         READ_CAPS(rdmacaps);
3013         READ_CAPS(iscsicaps);
3014         READ_CAPS(fcoecaps);
3015
3016         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
3017                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
3018                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
3019                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3020                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
3021                 if (rc != 0) {
3022                         device_printf(sc->dev,
3023                             "failed to query NIC parameters: %d.\n", rc);
3024                         return (rc);
3025                 }
3026                 sc->tids.etid_base = val[0];
3027                 sc->params.etid_min = val[0];
3028                 sc->tids.netids = val[1] - val[0] + 1;
3029                 sc->params.netids = sc->tids.netids;
3030                 sc->params.eo_wr_cred = val[2];
3031                 sc->params.ethoffload = 1;
3032         }
3033
3034         if (sc->toecaps) {
3035                 /* query offload-related parameters */
3036                 param[0] = FW_PARAM_DEV(NTID);
3037                 param[1] = FW_PARAM_PFVF(SERVER_START);
3038                 param[2] = FW_PARAM_PFVF(SERVER_END);
3039                 param[3] = FW_PARAM_PFVF(TDDP_START);
3040                 param[4] = FW_PARAM_PFVF(TDDP_END);
3041                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3042                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3043                 if (rc != 0) {
3044                         device_printf(sc->dev,
3045                             "failed to query TOE parameters: %d.\n", rc);
3046                         return (rc);
3047                 }
3048                 sc->tids.ntids = val[0];
3049                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
3050                 sc->tids.stid_base = val[1];
3051                 sc->tids.nstids = val[2] - val[1] + 1;
3052                 sc->vres.ddp.start = val[3];
3053                 sc->vres.ddp.size = val[4] - val[3] + 1;
3054                 sc->params.ofldq_wr_cred = val[5];
3055                 sc->params.offload = 1;
3056         }
3057         if (sc->rdmacaps) {
3058                 param[0] = FW_PARAM_PFVF(STAG_START);
3059                 param[1] = FW_PARAM_PFVF(STAG_END);
3060                 param[2] = FW_PARAM_PFVF(RQ_START);
3061                 param[3] = FW_PARAM_PFVF(RQ_END);
3062                 param[4] = FW_PARAM_PFVF(PBL_START);
3063                 param[5] = FW_PARAM_PFVF(PBL_END);
3064                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3065                 if (rc != 0) {
3066                         device_printf(sc->dev,
3067                             "failed to query RDMA parameters(1): %d.\n", rc);
3068                         return (rc);
3069                 }
3070                 sc->vres.stag.start = val[0];
3071                 sc->vres.stag.size = val[1] - val[0] + 1;
3072                 sc->vres.rq.start = val[2];
3073                 sc->vres.rq.size = val[3] - val[2] + 1;
3074                 sc->vres.pbl.start = val[4];
3075                 sc->vres.pbl.size = val[5] - val[4] + 1;
3076
3077                 param[0] = FW_PARAM_PFVF(SQRQ_START);
3078                 param[1] = FW_PARAM_PFVF(SQRQ_END);
3079                 param[2] = FW_PARAM_PFVF(CQ_START);
3080                 param[3] = FW_PARAM_PFVF(CQ_END);
3081                 param[4] = FW_PARAM_PFVF(OCQ_START);
3082                 param[5] = FW_PARAM_PFVF(OCQ_END);
3083                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3084                 if (rc != 0) {
3085                         device_printf(sc->dev,
3086                             "failed to query RDMA parameters(2): %d.\n", rc);
3087                         return (rc);
3088                 }
3089                 sc->vres.qp.start = val[0];
3090                 sc->vres.qp.size = val[1] - val[0] + 1;
3091                 sc->vres.cq.start = val[2];
3092                 sc->vres.cq.size = val[3] - val[2] + 1;
3093                 sc->vres.ocq.start = val[4];
3094                 sc->vres.ocq.size = val[5] - val[4] + 1;
3095         }
3096         if (sc->iscsicaps) {
3097                 param[0] = FW_PARAM_PFVF(ISCSI_START);
3098                 param[1] = FW_PARAM_PFVF(ISCSI_END);
3099                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3100                 if (rc != 0) {
3101                         device_printf(sc->dev,
3102                             "failed to query iSCSI parameters: %d.\n", rc);
3103                         return (rc);
3104                 }
3105                 sc->vres.iscsi.start = val[0];
3106                 sc->vres.iscsi.size = val[1] - val[0] + 1;
3107         }
3108
3109         /*
3110          * We've got the params we wanted to query via the firmware.  Now grab
3111          * some others directly from the chip.
3112          */
3113         rc = t4_read_chip_settings(sc);
3114
3115         return (rc);
3116 }
3117
3118 static int
3119 set_params__post_init(struct adapter *sc)
3120 {
3121         uint32_t param, val;
3122
3123         /* ask for encapsulated CPLs */
3124         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3125         val = 1;
3126         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3127
3128         return (0);
3129 }
3130
3131 #undef FW_PARAM_PFVF
3132 #undef FW_PARAM_DEV
3133
3134 static void
3135 t4_set_desc(struct adapter *sc)
3136 {
3137         char buf[128];
3138         struct adapter_params *p = &sc->params;
3139
3140         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
3141             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
3142             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
3143
3144         device_set_desc_copy(sc->dev, buf);
3145 }
3146
3147 static void
3148 build_medialist(struct port_info *pi, struct ifmedia *media)
3149 {
3150         int m;
3151
3152         PORT_LOCK(pi);
3153
3154         ifmedia_removeall(media);
3155
3156         m = IFM_ETHER | IFM_FDX;
3157
3158         switch(pi->port_type) {
3159         case FW_PORT_TYPE_BT_XFI:
3160         case FW_PORT_TYPE_BT_XAUI:
3161                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
3162                 /* fall through */
3163
3164         case FW_PORT_TYPE_BT_SGMII:
3165                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
3166                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
3167                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
3168                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
3169                 break;
3170
3171         case FW_PORT_TYPE_CX4:
3172                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
3173                 ifmedia_set(media, m | IFM_10G_CX4);
3174                 break;
3175
3176         case FW_PORT_TYPE_QSFP_10G:
3177         case FW_PORT_TYPE_SFP:
3178         case FW_PORT_TYPE_FIBER_XFI:
3179         case FW_PORT_TYPE_FIBER_XAUI:
3180                 switch (pi->mod_type) {
3181
3182                 case FW_PORT_MOD_TYPE_LR:
3183                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
3184                         ifmedia_set(media, m | IFM_10G_LR);
3185                         break;
3186
3187                 case FW_PORT_MOD_TYPE_SR:
3188                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
3189                         ifmedia_set(media, m | IFM_10G_SR);
3190                         break;
3191
3192                 case FW_PORT_MOD_TYPE_LRM:
3193                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
3194                         ifmedia_set(media, m | IFM_10G_LRM);
3195                         break;
3196
3197                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3198                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3199                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
3200                         ifmedia_set(media, m | IFM_10G_TWINAX);
3201                         break;
3202
3203                 case FW_PORT_MOD_TYPE_NONE:
3204                         m &= ~IFM_FDX;
3205                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3206                         ifmedia_set(media, m | IFM_NONE);
3207                         break;
3208
3209                 case FW_PORT_MOD_TYPE_NA:
3210                 case FW_PORT_MOD_TYPE_ER:
3211                 default:
3212                         device_printf(pi->dev,
3213                             "unknown port_type (%d), mod_type (%d)\n",
3214                             pi->port_type, pi->mod_type);
3215                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3216                         ifmedia_set(media, m | IFM_UNKNOWN);
3217                         break;
3218                 }
3219                 break;
3220
3221         case FW_PORT_TYPE_QSFP:
3222                 switch (pi->mod_type) {
3223
3224                 case FW_PORT_MOD_TYPE_LR:
3225                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
3226                         ifmedia_set(media, m | IFM_40G_LR4);
3227                         break;
3228
3229                 case FW_PORT_MOD_TYPE_SR:
3230                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
3231                         ifmedia_set(media, m | IFM_40G_SR4);
3232                         break;
3233
3234                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3235                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3236                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
3237                         ifmedia_set(media, m | IFM_40G_CR4);
3238                         break;
3239
3240                 case FW_PORT_MOD_TYPE_NONE:
3241                         m &= ~IFM_FDX;
3242                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3243                         ifmedia_set(media, m | IFM_NONE);
3244                         break;
3245
3246                 default:
3247                         device_printf(pi->dev,
3248                             "unknown port_type (%d), mod_type (%d)\n",
3249                             pi->port_type, pi->mod_type);
3250                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3251                         ifmedia_set(media, m | IFM_UNKNOWN);
3252                         break;
3253                 }
3254                 break;
3255
3256         default:
3257                 device_printf(pi->dev,
3258                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3259                     pi->mod_type);
3260                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3261                 ifmedia_set(media, m | IFM_UNKNOWN);
3262                 break;
3263         }
3264
3265         PORT_UNLOCK(pi);
3266 }
3267
3268 #define FW_MAC_EXACT_CHUNK      7
3269
3270 /*
3271  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3272  * indicates which parameters should be programmed (the rest are left alone).
3273  */
3274 int
3275 update_mac_settings(struct ifnet *ifp, int flags)
3276 {
3277         int rc = 0;
3278         struct vi_info *vi = ifp->if_softc;
3279         struct port_info *pi = vi->pi;
3280         struct adapter *sc = pi->adapter;
3281         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3282
3283         ASSERT_SYNCHRONIZED_OP(sc);
3284         KASSERT(flags, ("%s: not told what to update.", __func__));
3285
3286         if (flags & XGMAC_MTU)
3287                 mtu = ifp->if_mtu;
3288
3289         if (flags & XGMAC_PROMISC)
3290                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3291
3292         if (flags & XGMAC_ALLMULTI)
3293                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3294
3295         if (flags & XGMAC_VLANEX)
3296                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3297
3298         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3299                 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
3300                     allmulti, 1, vlanex, false);
3301                 if (rc) {
3302                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3303                             rc);
3304                         return (rc);
3305                 }
3306         }
3307
3308         if (flags & XGMAC_UCADDR) {
3309                 uint8_t ucaddr[ETHER_ADDR_LEN];
3310
3311                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3312                 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
3313                     ucaddr, true, true);
3314                 if (rc < 0) {
3315                         rc = -rc;
3316                         if_printf(ifp, "change_mac failed: %d\n", rc);
3317                         return (rc);
3318                 } else {
3319                         vi->xact_addr_filt = rc;
3320                         rc = 0;
3321                 }
3322         }
3323
3324         if (flags & XGMAC_MCADDRS) {
3325                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3326                 int del = 1;
3327                 uint64_t hash = 0;
3328                 struct ifmultiaddr *ifma;
3329                 int i = 0, j;
3330
3331                 if_maddr_rlock(ifp);
3332                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3333                         if (ifma->ifma_addr->sa_family != AF_LINK)
3334                                 continue;
3335                         mcaddr[i] =
3336                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3337                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3338                         i++;
3339
3340                         if (i == FW_MAC_EXACT_CHUNK) {
3341                                 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
3342                                     del, i, mcaddr, NULL, &hash, 0);
3343                                 if (rc < 0) {
3344                                         rc = -rc;
3345                                         for (j = 0; j < i; j++) {
3346                                                 if_printf(ifp,
3347                                                     "failed to add mc address"
3348                                                     " %02x:%02x:%02x:"
3349                                                     "%02x:%02x:%02x rc=%d\n",
3350                                                     mcaddr[j][0], mcaddr[j][1],
3351                                                     mcaddr[j][2], mcaddr[j][3],
3352                                                     mcaddr[j][4], mcaddr[j][5],
3353                                                     rc);
3354                                         }
3355                                         goto mcfail;
3356                                 }
3357                                 del = 0;
3358                                 i = 0;
3359                         }
3360                 }
3361                 if (i > 0) {
3362                         rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
3363                             mcaddr, NULL, &hash, 0);
3364                         if (rc < 0) {
3365                                 rc = -rc;
3366                                 for (j = 0; j < i; j++) {
3367                                         if_printf(ifp,
3368                                             "failed to add mc address"
3369                                             " %02x:%02x:%02x:"
3370                                             "%02x:%02x:%02x rc=%d\n",
3371                                             mcaddr[j][0], mcaddr[j][1],
3372                                             mcaddr[j][2], mcaddr[j][3],
3373                                             mcaddr[j][4], mcaddr[j][5],
3374                                             rc);
3375                                 }
3376                                 goto mcfail;
3377                         }
3378                 }
3379
3380                 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
3381                 if (rc != 0)
3382                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3383 mcfail:
3384                 if_maddr_runlock(ifp);
3385         }
3386
3387         return (rc);
3388 }
3389
3390 /*
3391  * {begin|end}_synchronized_op must be called from the same thread.
3392  */
3393 int
3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
3395     char *wmesg)
3396 {
3397         int rc, pri;
3398
3399 #ifdef WITNESS
3400         /* the caller thinks it's ok to sleep, but is it really? */
3401         if (flags & SLEEP_OK)
3402                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
3403                     "begin_synchronized_op");
3404 #endif
3405
3406         if (INTR_OK)
3407                 pri = PCATCH;
3408         else
3409                 pri = 0;
3410
3411         ADAPTER_LOCK(sc);
3412         for (;;) {
3413
3414                 if (vi && IS_DOOMED(vi)) {
3415                         rc = ENXIO;
3416                         goto done;
3417                 }
3418
3419                 if (!IS_BUSY(sc)) {
3420                         rc = 0;
3421                         break;
3422                 }
3423
3424                 if (!(flags & SLEEP_OK)) {
3425                         rc = EBUSY;
3426                         goto done;
3427                 }
3428
3429                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3430                         rc = EINTR;
3431                         goto done;
3432                 }
3433         }
3434
3435         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3436         SET_BUSY(sc);
3437 #ifdef INVARIANTS
3438         sc->last_op = wmesg;
3439         sc->last_op_thr = curthread;
3440         sc->last_op_flags = flags;
3441 #endif
3442
3443 done:
3444         if (!(flags & HOLD_LOCK) || rc)
3445                 ADAPTER_UNLOCK(sc);
3446
3447         return (rc);
3448 }
3449
3450 /*
3451  * Tell if_ioctl and if_init that the VI is going away.  This is
3452  * special variant of begin_synchronized_op and must be paired with a
3453  * call to end_synchronized_op.
3454  */
3455 void
3456 doom_vi(struct adapter *sc, struct vi_info *vi)
3457 {
3458
3459         ADAPTER_LOCK(sc);
3460         SET_DOOMED(vi);
3461         wakeup(&sc->flags);
3462         while (IS_BUSY(sc))
3463                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
3464         SET_BUSY(sc);
3465 #ifdef INVARIANTS
3466         sc->last_op = "t4detach";
3467         sc->last_op_thr = curthread;
3468         sc->last_op_flags = 0;
3469 #endif
3470         ADAPTER_UNLOCK(sc);
3471 }
3472
3473 /*
3474  * {begin|end}_synchronized_op must be called from the same thread.
3475  */
3476 void
3477 end_synchronized_op(struct adapter *sc, int flags)
3478 {
3479
3480         if (flags & LOCK_HELD)
3481                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3482         else
3483                 ADAPTER_LOCK(sc);
3484
3485         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3486         CLR_BUSY(sc);
3487         wakeup(&sc->flags);
3488         ADAPTER_UNLOCK(sc);
3489 }
3490
3491 static int
3492 cxgbe_init_synchronized(struct vi_info *vi)
3493 {
3494         struct port_info *pi = vi->pi;
3495         struct adapter *sc = pi->adapter;
3496         struct ifnet *ifp = vi->ifp;
3497         int rc = 0, i;
3498         struct sge_txq *txq;
3499
3500         ASSERT_SYNCHRONIZED_OP(sc);
3501
3502         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3503                 return (0);     /* already running */
3504
3505         if (!(sc->flags & FULL_INIT_DONE) &&
3506             ((rc = adapter_full_init(sc)) != 0))
3507                 return (rc);    /* error message displayed already */
3508
3509         if (!(vi->flags & VI_INIT_DONE) &&
3510             ((rc = vi_full_init(vi)) != 0))
3511                 return (rc); /* error message displayed already */
3512
3513         rc = update_mac_settings(ifp, XGMAC_ALL);
3514         if (rc)
3515                 goto done;      /* error message displayed already */
3516
3517         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
3518         if (rc != 0) {
3519                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3520                 goto done;
3521         }
3522
3523         /*
3524          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3525          * if this changes.
3526          */
3527
3528         for_each_txq(vi, i, txq) {
3529                 TXQ_LOCK(txq);
3530                 txq->eq.flags |= EQ_ENABLED;
3531                 TXQ_UNLOCK(txq);
3532         }
3533
3534         /*
3535          * The first iq of the first port to come up is used for tracing.
3536          */
3537         if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
3538                 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
3539                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3540                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3541                     V_QUEUENUMBER(sc->traceq));
3542                 pi->flags |= HAS_TRACEQ;
3543         }
3544
3545         /* all ok */
3546         PORT_LOCK(pi);
3547         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3548         pi->up_vis++;
3549
3550         if (pi->nvi > 1)
3551                 callout_reset(&vi->tick, hz, vi_tick, vi);
3552         else
3553                 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3554         PORT_UNLOCK(pi);
3555 done:
3556         if (rc != 0)
3557                 cxgbe_uninit_synchronized(vi);
3558
3559         return (rc);
3560 }
3561
3562 /*
3563  * Idempotent.
3564  */
3565 static int
3566 cxgbe_uninit_synchronized(struct vi_info *vi)
3567 {
3568         struct port_info *pi = vi->pi;
3569         struct adapter *sc = pi->adapter;
3570         struct ifnet *ifp = vi->ifp;
3571         int rc, i;
3572         struct sge_txq *txq;
3573
3574         ASSERT_SYNCHRONIZED_OP(sc);
3575
3576         if (!(vi->flags & VI_INIT_DONE)) {
3577                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3578                     ("uninited VI is running"));
3579                 return (0);
3580         }
3581
3582         /*
3583          * Disable the VI so that all its data in either direction is discarded
3584          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3585          * tick) intact as the TP can deliver negative advice or data that it's
3586          * holding in its RAM (for an offloaded connection) even after the VI is
3587          * disabled.
3588          */
3589         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
3590         if (rc) {
3591                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3592                 return (rc);
3593         }
3594
3595         for_each_txq(vi, i, txq) {
3596                 TXQ_LOCK(txq);
3597                 txq->eq.flags &= ~EQ_ENABLED;
3598                 TXQ_UNLOCK(txq);
3599         }
3600
3601         PORT_LOCK(pi);
3602         if (pi->nvi == 1)
3603                 callout_stop(&pi->tick);
3604         else
3605                 callout_stop(&vi->tick);
3606         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3607                 PORT_UNLOCK(pi);
3608                 return (0);
3609         }
3610         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3611         pi->up_vis--;
3612         if (pi->up_vis > 0) {
3613                 PORT_UNLOCK(pi);
3614                 return (0);
3615         }
3616         PORT_UNLOCK(pi);
3617
3618         pi->link_cfg.link_ok = 0;
3619         pi->link_cfg.speed = 0;
3620         pi->linkdnrc = -1;
3621         t4_os_link_changed(sc, pi->port_id, 0, -1);
3622
3623         return (0);
3624 }
3625
3626 /*
3627  * It is ok for this function to fail midway and return right away.  t4_detach
3628  * will walk the entire sc->irq list and clean up whatever is valid.
3629  */
3630 static int
3631 setup_intr_handlers(struct adapter *sc)
3632 {
3633         int rc, rid, p, q, v;
3634         char s[8];
3635         struct irq *irq;
3636         struct port_info *pi;
3637         struct vi_info *vi;
3638         struct sge_rxq *rxq;
3639 #ifdef TCP_OFFLOAD
3640         struct sge_ofld_rxq *ofld_rxq;
3641 #endif
3642 #ifdef DEV_NETMAP
3643         struct sge_nm_rxq *nm_rxq;
3644 #endif
3645 #ifdef RSS
3646         int nbuckets = rss_getnumbuckets();
3647 #endif
3648
3649         /*
3650          * Setup interrupts.
3651          */
3652         irq = &sc->irq[0];
3653         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3654         if (sc->intr_count == 1)
3655                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3656
3657         /* Multiple interrupts. */
3658         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3659             ("%s: too few intr.", __func__));
3660
3661         /* The first one is always error intr */
3662         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3663         if (rc != 0)
3664                 return (rc);
3665         irq++;
3666         rid++;
3667
3668         /* The second one is always the firmware event queue */
3669         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3670         if (rc != 0)
3671                 return (rc);
3672         irq++;
3673         rid++;
3674
3675         for_each_port(sc, p) {
3676                 pi = sc->port[p];
3677                 for_each_vi(pi, v, vi) {
3678                         vi->first_intr = rid - 1;
3679 #ifdef DEV_NETMAP
3680                         if (vi->flags & VI_NETMAP) {
3681                                 for_each_nm_rxq(vi, q, nm_rxq) {
3682                                         snprintf(s, sizeof(s), "%d-%d", p, q);
3683                                         rc = t4_alloc_irq(sc, irq, rid,
3684                                             t4_nm_intr, nm_rxq, s);
3685                                         if (rc != 0)
3686                                                 return (rc);
3687                                         irq++;
3688                                         rid++;
3689                                         vi->nintr++;
3690                                 }
3691                                 continue;
3692                         }
3693 #endif
3694                         if (vi->flags & INTR_RXQ) {
3695                                 for_each_rxq(vi, q, rxq) {
3696                                         if (v == 0)
3697                                                 snprintf(s, sizeof(s), "%d.%d",
3698                                                     p, q);
3699                                         else
3700                                                 snprintf(s, sizeof(s),
3701                                                     "%d(%d).%d", p, v, q);
3702                                         rc = t4_alloc_irq(sc, irq, rid,
3703                                             t4_intr, rxq, s);
3704                                         if (rc != 0)
3705                                                 return (rc);
3706 #ifdef RSS
3707                                         bus_bind_intr(sc->dev, irq->res,
3708                                             rss_getcpu(q % nbuckets));
3709 #endif
3710                                         irq++;
3711                                         rid++;
3712                                         vi->nintr++;
3713                                 }
3714                         }
3715 #ifdef TCP_OFFLOAD
3716                         if (vi->flags & INTR_OFLD_RXQ) {
3717                                 for_each_ofld_rxq(vi, q, ofld_rxq) {
3718                                         snprintf(s, sizeof(s), "%d,%d", p, q);
3719                                         rc = t4_alloc_irq(sc, irq, rid,
3720                                             t4_intr, ofld_rxq, s);
3721                                         if (rc != 0)
3722                                                 return (rc);
3723                                         irq++;
3724                                         rid++;
3725                                         vi->nintr++;
3726                                 }
3727                         }
3728 #endif
3729                 }
3730         }
3731         MPASS(irq == &sc->irq[sc->intr_count]);
3732
3733         return (0);
3734 }
3735
3736 int
3737 adapter_full_init(struct adapter *sc)
3738 {
3739         int rc, i;
3740
3741         ASSERT_SYNCHRONIZED_OP(sc);
3742         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3743         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3744             ("%s: FULL_INIT_DONE already", __func__));
3745
3746         /*
3747          * queues that belong to the adapter (not any particular port).
3748          */
3749         rc = t4_setup_adapter_queues(sc);
3750         if (rc != 0)
3751                 goto done;
3752
3753         for (i = 0; i < nitems(sc->tq); i++) {
3754                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3755                     taskqueue_thread_enqueue, &sc->tq[i]);
3756                 if (sc->tq[i] == NULL) {
3757                         device_printf(sc->dev,
3758                             "failed to allocate task queue %d\n", i);
3759                         rc = ENOMEM;
3760                         goto done;
3761                 }
3762                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3763                     device_get_nameunit(sc->dev), i);
3764         }
3765
3766         t4_intr_enable(sc);
3767         sc->flags |= FULL_INIT_DONE;
3768 done:
3769         if (rc != 0)
3770                 adapter_full_uninit(sc);
3771
3772         return (rc);
3773 }
3774
3775 int
3776 adapter_full_uninit(struct adapter *sc)
3777 {
3778         int i;
3779
3780         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3781
3782         t4_teardown_adapter_queues(sc);
3783
3784         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3785                 taskqueue_free(sc->tq[i]);
3786                 sc->tq[i] = NULL;
3787         }
3788
3789         sc->flags &= ~FULL_INIT_DONE;
3790
3791         return (0);
3792 }
3793
3794 #ifdef RSS
3795 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
3796     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
3797     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
3798     RSS_HASHTYPE_RSS_UDP_IPV6)
3799
3800 /* Translates kernel hash types to hardware. */
3801 static int
3802 hashconfig_to_hashen(int hashconfig)
3803 {
3804         int hashen = 0;
3805
3806         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
3807                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
3808         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
3809                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
3810         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
3811                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3812                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3813         }
3814         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
3815                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3816                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3817         }
3818         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
3819                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3820         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
3821                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3822
3823         return (hashen);
3824 }
3825
3826 /* Translates hardware hash types to kernel. */
3827 static int
3828 hashen_to_hashconfig(int hashen)
3829 {
3830         int hashconfig = 0;
3831
3832         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
3833                 /*
3834                  * If UDP hashing was enabled it must have been enabled for
3835                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
3836                  * enabling any 4-tuple hash is nonsense configuration.
3837                  */
3838                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3839                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
3840
3841                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3842                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
3843                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3844                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
3845         }
3846         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3847                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
3848         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3849                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
3850         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
3851                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
3852         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
3853                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
3854
3855         return (hashconfig);
3856 }
3857 #endif
3858
3859 int
3860 vi_full_init(struct vi_info *vi)
3861 {
3862         struct adapter *sc = vi->pi->adapter;
3863         struct ifnet *ifp = vi->ifp;
3864         uint16_t *rss;
3865         struct sge_rxq *rxq;
3866         int rc, i, j, hashen;
3867 #ifdef RSS
3868         int nbuckets = rss_getnumbuckets();
3869         int hashconfig = rss_gethashconfig();
3870         int extra;
3871         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3872         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3873 #endif
3874
3875         ASSERT_SYNCHRONIZED_OP(sc);
3876         KASSERT((vi->flags & VI_INIT_DONE) == 0,
3877             ("%s: VI_INIT_DONE already", __func__));
3878
3879         sysctl_ctx_init(&vi->ctx);
3880         vi->flags |= VI_SYSCTL_CTX;
3881
3882         /*
3883          * Allocate tx/rx/fl queues for this VI.
3884          */
3885         rc = t4_setup_vi_queues(vi);
3886         if (rc != 0)
3887                 goto done;      /* error message displayed already */
3888
3889 #ifdef DEV_NETMAP
3890         /* Netmap VIs configure RSS when netmap is enabled. */
3891         if (vi->flags & VI_NETMAP) {
3892                 vi->flags |= VI_INIT_DONE;
3893                 return (0);
3894         }
3895 #endif
3896
3897         /*
3898          * Setup RSS for this VI.  Save a copy of the RSS table for later use.
3899          */
3900         if (vi->nrxq > vi->rss_size) {
3901                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
3902                     "some queues will never receive traffic.\n", vi->nrxq,
3903                     vi->rss_size);
3904         } else if (vi->rss_size % vi->nrxq) {
3905                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
3906                     "expect uneven traffic distribution.\n", vi->nrxq,
3907                     vi->rss_size);
3908         }
3909 #ifdef RSS
3910         MPASS(RSS_KEYSIZE == 40);
3911         if (vi->nrxq != nbuckets) {
3912                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
3913                     "performance will be impacted.\n", vi->nrxq, nbuckets);
3914         }
3915
3916         rss_getkey((void *)&raw_rss_key[0]);
3917         for (i = 0; i < nitems(rss_key); i++) {
3918                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
3919         }
3920         t4_write_rss_key(sc, (void *)&rss_key[0], -1);
3921 #endif
3922         rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3923         for (i = 0; i < vi->rss_size;) {
3924 #ifdef RSS
3925                 j = rss_get_indirection_to_bucket(i);
3926                 j %= vi->nrxq;
3927                 rxq = &sc->sge.rxq[vi->first_rxq + j];
3928                 rss[i++] = rxq->iq.abs_id;
3929 #else
3930                 for_each_rxq(vi, j, rxq) {
3931                         rss[i++] = rxq->iq.abs_id;
3932                         if (i == vi->rss_size)
3933                                 break;
3934                 }
3935 #endif
3936         }
3937
3938         rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
3939             vi->rss_size);
3940         if (rc != 0) {
3941                 if_printf(ifp, "rss_config failed: %d\n", rc);
3942                 goto done;
3943         }
3944
3945 #ifdef RSS
3946         hashen = hashconfig_to_hashen(hashconfig);
3947
3948         /*
3949          * We may have had to enable some hashes even though the global config
3950          * wants them disabled.  This is a potential problem that must be
3951          * reported to the user.
3952          */
3953         extra = hashen_to_hashconfig(hashen) ^ hashconfig;
3954
3955         /*
3956          * If we consider only the supported hash types, then the enabled hashes
3957          * are a superset of the requested hashes.  In other words, there cannot
3958          * be any supported hash that was requested but not enabled, but there
3959          * can be hashes that were not requested but had to be enabled.
3960          */
3961         extra &= SUPPORTED_RSS_HASHTYPES;
3962         MPASS((extra & hashconfig) == 0);
3963
3964         if (extra) {
3965                 if_printf(ifp,
3966                     "global RSS config (0x%x) cannot be accomodated.\n",
3967                     hashconfig);
3968         }
3969         if (extra & RSS_HASHTYPE_RSS_IPV4)
3970                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
3971         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
3972                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
3973         if (extra & RSS_HASHTYPE_RSS_IPV6)
3974                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
3975         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
3976                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
3977         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
3978                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
3979         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
3980                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
3981 #else
3982         hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
3983             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
3984             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3985             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
3986 #endif
3987         rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]);
3988         if (rc != 0) {
3989                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
3990                 goto done;
3991         }
3992
3993         vi->rss = rss;
3994         vi->flags |= VI_INIT_DONE;
3995 done:
3996         if (rc != 0)
3997                 vi_full_uninit(vi);
3998
3999         return (rc);
4000 }
4001
4002 /*
4003  * Idempotent.
4004  */
4005 int
4006 vi_full_uninit(struct vi_info *vi)
4007 {
4008         struct port_info *pi = vi->pi;
4009         struct adapter *sc = pi->adapter;
4010         int i;
4011         struct sge_rxq *rxq;
4012         struct sge_txq *txq;
4013 #ifdef TCP_OFFLOAD
4014         struct sge_ofld_rxq *ofld_rxq;
4015         struct sge_wrq *ofld_txq;
4016 #endif
4017
4018         if (vi->flags & VI_INIT_DONE) {
4019
4020                 /* Need to quiesce queues.  */
4021 #ifdef DEV_NETMAP
4022                 if (vi->flags & VI_NETMAP)
4023                         goto skip;
4024 #endif
4025
4026                 /* XXX: Only for the first VI? */
4027                 if (IS_MAIN_VI(vi))
4028                         quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
4029
4030                 for_each_txq(vi, i, txq) {
4031                         quiesce_txq(sc, txq);
4032                 }
4033
4034 #ifdef TCP_OFFLOAD
4035                 for_each_ofld_txq(vi, i, ofld_txq) {
4036                         quiesce_wrq(sc, ofld_txq);
4037                 }
4038 #endif
4039
4040                 for_each_rxq(vi, i, rxq) {
4041                         quiesce_iq(sc, &rxq->iq);
4042                         quiesce_fl(sc, &rxq->fl);
4043                 }
4044
4045 #ifdef TCP_OFFLOAD
4046                 for_each_ofld_rxq(vi, i, ofld_rxq) {
4047                         quiesce_iq(sc, &ofld_rxq->iq);
4048                         quiesce_fl(sc, &ofld_rxq->fl);
4049                 }
4050 #endif
4051                 free(vi->rss, M_CXGBE);
4052         }
4053 #ifdef DEV_NETMAP
4054 skip:
4055 #endif
4056
4057         t4_teardown_vi_queues(vi);
4058         vi->flags &= ~VI_INIT_DONE;
4059
4060         return (0);
4061 }
4062
4063 static void
4064 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
4065 {
4066         struct sge_eq *eq = &txq->eq;
4067         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4068
4069         (void) sc;      /* unused */
4070
4071 #ifdef INVARIANTS
4072         TXQ_LOCK(txq);
4073         MPASS((eq->flags & EQ_ENABLED) == 0);
4074         TXQ_UNLOCK(txq);
4075 #endif
4076
4077         /* Wait for the mp_ring to empty. */
4078         while (!mp_ring_is_idle(txq->r)) {
4079                 mp_ring_check_drainage(txq->r, 0);
4080                 pause("rquiesce", 1);
4081         }
4082
4083         /* Then wait for the hardware to finish. */
4084         while (spg->cidx != htobe16(eq->pidx))
4085                 pause("equiesce", 1);
4086
4087         /* Finally, wait for the driver to reclaim all descriptors. */
4088         while (eq->cidx != eq->pidx)
4089                 pause("dquiesce", 1);
4090 }
4091
4092 static void
4093 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
4094 {
4095
4096         /* XXXTX */
4097 }
4098
4099 static void
4100 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
4101 {
4102         (void) sc;      /* unused */
4103
4104         /* Synchronize with the interrupt handler */
4105         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
4106                 pause("iqfree", 1);
4107 }
4108
4109 static void
4110 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
4111 {
4112         mtx_lock(&sc->sfl_lock);
4113         FL_LOCK(fl);
4114         fl->flags |= FL_DOOMED;
4115         FL_UNLOCK(fl);
4116         callout_stop(&sc->sfl_callout);
4117         mtx_unlock(&sc->sfl_lock);
4118
4119         KASSERT((fl->flags & FL_STARVING) == 0,
4120             ("%s: still starving", __func__));
4121 }
4122
4123 static int
4124 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
4125     driver_intr_t *handler, void *arg, char *name)
4126 {
4127         int rc;
4128
4129         irq->rid = rid;
4130         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
4131             RF_SHAREABLE | RF_ACTIVE);
4132         if (irq->res == NULL) {
4133                 device_printf(sc->dev,
4134                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
4135                 return (ENOMEM);
4136         }
4137
4138         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
4139             NULL, handler, arg, &irq->tag);
4140         if (rc != 0) {
4141                 device_printf(sc->dev,
4142                     "failed to setup interrupt for rid %d, name %s: %d\n",
4143                     rid, name, rc);
4144         } else if (name)
4145                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
4146
4147         return (rc);
4148 }
4149
4150 static int
4151 t4_free_irq(struct adapter *sc, struct irq *irq)
4152 {
4153         if (irq->tag)
4154                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
4155         if (irq->res)
4156                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
4157
4158         bzero(irq, sizeof(*irq));
4159
4160         return (0);
4161 }
4162
4163 static void
4164 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
4165     unsigned int end)
4166 {
4167         uint32_t *p = (uint32_t *)(buf + start);
4168
4169         for ( ; start <= end; start += sizeof(uint32_t))
4170                 *p++ = t4_read_reg(sc, start);
4171 }
4172
4173 static void
4174 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
4175 {
4176         int i, n;
4177         const unsigned int *reg_ranges;
4178         static const unsigned int t4_reg_ranges[] = {
4179                 0x1008, 0x1108,
4180                 0x1180, 0x11b4,
4181                 0x11fc, 0x123c,
4182                 0x1300, 0x173c,
4183                 0x1800, 0x18fc,
4184                 0x3000, 0x30d8,
4185                 0x30e0, 0x5924,
4186                 0x5960, 0x59d4,
4187                 0x5a00, 0x5af8,
4188                 0x6000, 0x6098,
4189                 0x6100, 0x6150,
4190                 0x6200, 0x6208,
4191                 0x6240, 0x6248,
4192                 0x6280, 0x6338,
4193                 0x6370, 0x638c,
4194                 0x6400, 0x643c,
4195                 0x6500, 0x6524,
4196                 0x6a00, 0x6a38,
4197                 0x6a60, 0x6a78,
4198                 0x6b00, 0x6b84,
4199                 0x6bf0, 0x6c84,
4200                 0x6cf0, 0x6d84,
4201                 0x6df0, 0x6e84,
4202                 0x6ef0, 0x6f84,
4203                 0x6ff0, 0x7084,
4204                 0x70f0, 0x7184,
4205                 0x71f0, 0x7284,
4206                 0x72f0, 0x7384,
4207                 0x73f0, 0x7450,
4208                 0x7500, 0x7530,
4209                 0x7600, 0x761c,
4210                 0x7680, 0x76cc,
4211                 0x7700, 0x7798,
4212                 0x77c0, 0x77fc,
4213                 0x7900, 0x79fc,
4214                 0x7b00, 0x7c38,
4215                 0x7d00, 0x7efc,
4216                 0x8dc0, 0x8e1c,
4217                 0x8e30, 0x8e78,
4218                 0x8ea0, 0x8f6c,
4219                 0x8fc0, 0x9074,
4220                 0x90fc, 0x90fc,
4221                 0x9400, 0x9458,
4222                 0x9600, 0x96bc,
4223                 0x9800, 0x9808,
4224                 0x9820, 0x983c,
4225                 0x9850, 0x9864,
4226                 0x9c00, 0x9c6c,
4227                 0x9c80, 0x9cec,
4228                 0x9d00, 0x9d6c,
4229                 0x9d80, 0x9dec,
4230                 0x9e00, 0x9e6c,
4231                 0x9e80, 0x9eec,
4232                 0x9f00, 0x9f6c,
4233                 0x9f80, 0x9fec,
4234                 0xd004, 0xd03c,
4235                 0xdfc0, 0xdfe0,
4236                 0xe000, 0xea7c,
4237                 0xf000, 0x11110,
4238                 0x11118, 0x11190,
4239                 0x19040, 0x1906c,
4240                 0x19078, 0x19080,
4241                 0x1908c, 0x19124,
4242                 0x19150, 0x191b0,
4243                 0x191d0, 0x191e8,
4244                 0x19238, 0x1924c,
4245                 0x193f8, 0x19474,
4246                 0x19490, 0x194f8,
4247                 0x19800, 0x19f30,
4248                 0x1a000, 0x1a06c,
4249                 0x1a0b0, 0x1a120,
4250                 0x1a128, 0x1a138,
4251                 0x1a190, 0x1a1c4,
4252                 0x1a1fc, 0x1a1fc,
4253                 0x1e040, 0x1e04c,
4254                 0x1e284, 0x1e28c,
4255                 0x1e2c0, 0x1e2c0,
4256                 0x1e2e0, 0x1e2e0,
4257                 0x1e300, 0x1e384,
4258                 0x1e3c0, 0x1e3c8,
4259                 0x1e440, 0x1e44c,
4260                 0x1e684, 0x1e68c,
4261                 0x1e6c0, 0x1e6c0,
4262                 0x1e6e0, 0x1e6e0,
4263                 0x1e700, 0x1e784,
4264                 0x1e7c0, 0x1e7c8,
4265                 0x1e840, 0x1e84c,
4266                 0x1ea84, 0x1ea8c,
4267                 0x1eac0, 0x1eac0,
4268                 0x1eae0, 0x1eae0,
4269                 0x1eb00, 0x1eb84,
4270                 0x1ebc0, 0x1ebc8,
4271                 0x1ec40, 0x1ec4c,
4272                 0x1ee84, 0x1ee8c,
4273                 0x1eec0, 0x1eec0,
4274                 0x1eee0, 0x1eee0,
4275                 0x1ef00, 0x1ef84,
4276                 0x1efc0, 0x1efc8,
4277                 0x1f040, 0x1f04c,
4278                 0x1f284, 0x1f28c,
4279                 0x1f2c0, 0x1f2c0,
4280                 0x1f2e0, 0x1f2e0,
4281                 0x1f300, 0x1f384,
4282                 0x1f3c0, 0x1f3c8,
4283                 0x1f440, 0x1f44c,
4284                 0x1f684, 0x1f68c,
4285                 0x1f6c0, 0x1f6c0,
4286                 0x1f6e0, 0x1f6e0,
4287                 0x1f700, 0x1f784,
4288                 0x1f7c0, 0x1f7c8,
4289                 0x1f840, 0x1f84c,
4290                 0x1fa84, 0x1fa8c,
4291                 0x1fac0, 0x1fac0,
4292                 0x1fae0, 0x1fae0,
4293                 0x1fb00, 0x1fb84,
4294                 0x1fbc0, 0x1fbc8,
4295                 0x1fc40, 0x1fc4c,
4296                 0x1fe84, 0x1fe8c,
4297                 0x1fec0, 0x1fec0,
4298                 0x1fee0, 0x1fee0,
4299                 0x1ff00, 0x1ff84,
4300                 0x1ffc0, 0x1ffc8,
4301                 0x20000, 0x2002c,
4302                 0x20100, 0x2013c,
4303                 0x20190, 0x201c8,
4304                 0x20200, 0x20318,
4305                 0x20400, 0x20528,
4306                 0x20540, 0x20614,
4307                 0x21000, 0x21040,
4308                 0x2104c, 0x21060,
4309                 0x210c0, 0x210ec,
4310                 0x21200, 0x21268,
4311                 0x21270, 0x21284,
4312                 0x212fc, 0x21388,
4313                 0x21400, 0x21404,
4314                 0x21500, 0x21518,
4315                 0x2152c, 0x2153c,
4316                 0x21550, 0x21554,
4317                 0x21600, 0x21600,
4318                 0x21608, 0x21628,
4319                 0x21630, 0x2163c,
4320                 0x21700, 0x2171c,
4321                 0x21780, 0x2178c,
4322                 0x21800, 0x21c38,
4323                 0x21c80, 0x21d7c,
4324                 0x21e00, 0x21e04,
4325                 0x22000, 0x2202c,
4326                 0x22100, 0x2213c,
4327                 0x22190, 0x221c8,
4328                 0x22200, 0x22318,
4329                 0x22400, 0x22528,
4330                 0x22540, 0x22614,
4331                 0x23000, 0x23040,
4332                 0x2304c, 0x23060,
4333                 0x230c0, 0x230ec,
4334                 0x23200, 0x23268,
4335                 0x23270, 0x23284,
4336                 0x232fc, 0x23388,
4337                 0x23400, 0x23404,
4338                 0x23500, 0x23518,
4339                 0x2352c, 0x2353c,
4340                 0x23550, 0x23554,
4341                 0x23600, 0x23600,
4342                 0x23608, 0x23628,
4343                 0x23630, 0x2363c,
4344                 0x23700, 0x2371c,
4345                 0x23780, 0x2378c,
4346                 0x23800, 0x23c38,
4347                 0x23c80, 0x23d7c,
4348                 0x23e00, 0x23e04,
4349                 0x24000, 0x2402c,
4350                 0x24100, 0x2413c,
4351                 0x24190, 0x241c8,
4352                 0x24200, 0x24318,
4353                 0x24400, 0x24528,
4354                 0x24540, 0x24614,
4355                 0x25000, 0x25040,
4356                 0x2504c, 0x25060,
4357                 0x250c0, 0x250ec,
4358                 0x25200, 0x25268,
4359                 0x25270, 0x25284,
4360                 0x252fc, 0x25388,
4361                 0x25400, 0x25404,
4362                 0x25500, 0x25518,
4363                 0x2552c, 0x2553c,
4364                 0x25550, 0x25554,
4365                 0x25600, 0x25600,
4366                 0x25608, 0x25628,
4367                 0x25630, 0x2563c,
4368                 0x25700, 0x2571c,
4369                 0x25780, 0x2578c,
4370                 0x25800, 0x25c38,
4371                 0x25c80, 0x25d7c,
4372                 0x25e00, 0x25e04,
4373                 0x26000, 0x2602c,
4374                 0x26100, 0x2613c,
4375                 0x26190, 0x261c8,
4376                 0x26200, 0x26318,
4377                 0x26400, 0x26528,
4378                 0x26540, 0x26614,
4379                 0x27000, 0x27040,
4380                 0x2704c, 0x27060,
4381                 0x270c0, 0x270ec,
4382                 0x27200, 0x27268,
4383                 0x27270, 0x27284,
4384                 0x272fc, 0x27388,
4385                 0x27400, 0x27404,
4386                 0x27500, 0x27518,
4387                 0x2752c, 0x2753c,
4388                 0x27550, 0x27554,
4389                 0x27600, 0x27600,
4390                 0x27608, 0x27628,
4391                 0x27630, 0x2763c,
4392                 0x27700, 0x2771c,
4393                 0x27780, 0x2778c,
4394                 0x27800, 0x27c38,
4395                 0x27c80, 0x27d7c,
4396                 0x27e00, 0x27e04
4397         };
4398         static const unsigned int t5_reg_ranges[] = {
4399                 0x1008, 0x1148,
4400                 0x1180, 0x11b4,
4401                 0x11fc, 0x123c,
4402                 0x1280, 0x173c,
4403                 0x1800, 0x18fc,
4404                 0x3000, 0x3028,
4405                 0x3060, 0x30d8,
4406                 0x30e0, 0x30fc,
4407                 0x3140, 0x357c,
4408                 0x35a8, 0x35cc,
4409                 0x35ec, 0x35ec,
4410                 0x3600, 0x5624,
4411                 0x56cc, 0x575c,
4412                 0x580c, 0x5814,
4413                 0x5890, 0x58bc,
4414                 0x5940, 0x59dc,
4415                 0x59fc, 0x5a18,
4416                 0x5a60, 0x5a9c,
4417                 0x5b94, 0x5bfc,
4418                 0x6000, 0x6040,
4419                 0x6058, 0x614c,
4420                 0x7700, 0x7798,
4421                 0x77c0, 0x78fc,
4422                 0x7b00, 0x7c54,
4423                 0x7d00, 0x7efc,
4424                 0x8dc0, 0x8de0,
4425                 0x8df8, 0x8e84,
4426                 0x8ea0, 0x8f84,
4427                 0x8fc0, 0x90f8,
4428                 0x9400, 0x9470,
4429                 0x9600, 0x96f4,
4430                 0x9800, 0x9808,
4431                 0x9820, 0x983c,
4432                 0x9850, 0x9864,
4433                 0x9c00, 0x9c6c,
4434                 0x9c80, 0x9cec,
4435                 0x9d00, 0x9d6c,
4436                 0x9d80, 0x9dec,
4437                 0x9e00, 0x9e6c,
4438                 0x9e80, 0x9eec,
4439                 0x9f00, 0x9f6c,
4440                 0x9f80, 0xa020,
4441                 0xd004, 0xd03c,
4442                 0xdfc0, 0xdfe0,
4443                 0xe000, 0x11088,
4444                 0x1109c, 0x11110,
4445                 0x11118, 0x1117c,
4446                 0x11190, 0x11204,
4447                 0x19040, 0x1906c,
4448                 0x19078, 0x19080,
4449                 0x1908c, 0x19124,
4450                 0x19150, 0x191b0,
4451                 0x191d0, 0x191e8,
4452                 0x19238, 0x19290,
4453                 0x193f8, 0x19474,
4454                 0x19490, 0x194cc,
4455                 0x194f0, 0x194f8,
4456                 0x19c00, 0x19c60,
4457                 0x19c94, 0x19e10,
4458                 0x19e50, 0x19f34,
4459                 0x19f40, 0x19f50,
4460                 0x19f90, 0x19fe4,
4461                 0x1a000, 0x1a06c,
4462                 0x1a0b0, 0x1a120,
4463                 0x1a128, 0x1a138,
4464                 0x1a190, 0x1a1c4,
4465                 0x1a1fc, 0x1a1fc,
4466                 0x1e008, 0x1e00c,
4467                 0x1e040, 0x1e04c,
4468                 0x1e284, 0x1e290,
4469                 0x1e2c0, 0x1e2c0,
4470                 0x1e2e0, 0x1e2e0,
4471                 0x1e300, 0x1e384,
4472                 0x1e3c0, 0x1e3c8,
4473                 0x1e408, 0x1e40c,
4474                 0x1e440, 0x1e44c,
4475                 0x1e684, 0x1e690,
4476                 0x1e6c0, 0x1e6c0,
4477                 0x1e6e0, 0x1e6e0,
4478                 0x1e700, 0x1e784,
4479                 0x1e7c0, 0x1e7c8,
4480                 0x1e808, 0x1e80c,
4481                 0x1e840, 0x1e84c,
4482                 0x1ea84, 0x1ea90,
4483                 0x1eac0, 0x1eac0,
4484                 0x1eae0, 0x1eae0,
4485                 0x1eb00, 0x1eb84,
4486                 0x1ebc0, 0x1ebc8,
4487                 0x1ec08, 0x1ec0c,
4488                 0x1ec40, 0x1ec4c,
4489                 0x1ee84, 0x1ee90,
4490                 0x1eec0, 0x1eec0,
4491                 0x1eee0, 0x1eee0,
4492                 0x1ef00, 0x1ef84,
4493                 0x1efc0, 0x1efc8,
4494                 0x1f008, 0x1f00c,
4495                 0x1f040, 0x1f04c,
4496                 0x1f284, 0x1f290,
4497                 0x1f2c0, 0x1f2c0,
4498                 0x1f2e0, 0x1f2e0,
4499                 0x1f300, 0x1f384,
4500                 0x1f3c0, 0x1f3c8,
4501                 0x1f408, 0x1f40c,
4502                 0x1f440, 0x1f44c,
4503                 0x1f684, 0x1f690,
4504                 0x1f6c0, 0x1f6c0,
4505                 0x1f6e0, 0x1f6e0,
4506                 0x1f700, 0x1f784,
4507                 0x1f7c0, 0x1f7c8,
4508                 0x1f808, 0x1f80c,
4509                 0x1f840, 0x1f84c,
4510                 0x1fa84, 0x1fa90,
4511                 0x1fac0, 0x1fac0,
4512                 0x1fae0, 0x1fae0,
4513                 0x1fb00, 0x1fb84,
4514                 0x1fbc0, 0x1fbc8,
4515                 0x1fc08, 0x1fc0c,
4516                 0x1fc40, 0x1fc4c,
4517                 0x1fe84, 0x1fe90,
4518                 0x1fec0, 0x1fec0,
4519                 0x1fee0, 0x1fee0,
4520                 0x1ff00, 0x1ff84,
4521                 0x1ffc0, 0x1ffc8,
4522                 0x30000, 0x30030,
4523                 0x30100, 0x30144,
4524                 0x30190, 0x301d0,
4525                 0x30200, 0x30318,
4526                 0x30400, 0x3052c,
4527                 0x30540, 0x3061c,
4528                 0x30800, 0x30834,
4529                 0x308c0, 0x30908,
4530                 0x30910, 0x309ac,
4531                 0x30a00, 0x30a2c,
4532                 0x30a44, 0x30a50,
4533                 0x30a74, 0x30c24,
4534                 0x30d00, 0x30d00,
4535                 0x30d08, 0x30d14,
4536                 0x30d1c, 0x30d20,
4537                 0x30d3c, 0x30d50,
4538                 0x31200, 0x3120c,
4539                 0x31220, 0x31220,
4540                 0x31240, 0x31240,
4541                 0x31600, 0x3160c,
4542                 0x31a00, 0x31a1c,
4543                 0x31e00, 0x31e20,
4544                 0x31e38, 0x31e3c,
4545                 0x31e80, 0x31e80,
4546                 0x31e88, 0x31ea8,
4547                 0x31eb0, 0x31eb4,
4548                 0x31ec8, 0x31ed4,
4549                 0x31fb8, 0x32004,
4550                 0x32200, 0x32200,
4551                 0x32208, 0x32240,
4552                 0x32248, 0x32280,
4553                 0x32288, 0x322c0,
4554                 0x322c8, 0x322fc,
4555                 0x32600, 0x32630,
4556                 0x32a00, 0x32abc,
4557                 0x32b00, 0x32b70,
4558                 0x33000, 0x33048,
4559                 0x33060, 0x3309c,
4560                 0x330f0, 0x33148,
4561                 0x33160, 0x3319c,
4562                 0x331f0, 0x332e4,
4563                 0x332f8, 0x333e4,
4564                 0x333f8, 0x33448,
4565                 0x33460, 0x3349c,
4566                 0x334f0, 0x33548,
4567                 0x33560, 0x3359c,
4568                 0x335f0, 0x336e4,
4569                 0x336f8, 0x337e4,
4570                 0x337f8, 0x337fc,
4571                 0x33814, 0x33814,
4572                 0x3382c, 0x3382c,
4573                 0x33880, 0x3388c,
4574                 0x338e8, 0x338ec,
4575                 0x33900, 0x33948,
4576                 0x33960, 0x3399c,
4577                 0x339f0, 0x33ae4,
4578                 0x33af8, 0x33b10,
4579                 0x33b28, 0x33b28,
4580                 0x33b3c, 0x33b50,
4581                 0x33bf0, 0x33c10,
4582                 0x33c28, 0x33c28,
4583                 0x33c3c, 0x33c50,
4584                 0x33cf0, 0x33cfc,
4585                 0x34000, 0x34030,
4586                 0x34100, 0x34144,
4587                 0x34190, 0x341d0,
4588                 0x34200, 0x34318,
4589                 0x34400, 0x3452c,
4590                 0x34540, 0x3461c,
4591                 0x34800, 0x34834,
4592                 0x348c0, 0x34908,
4593                 0x34910, 0x349ac,
4594                 0x34a00, 0x34a2c,
4595                 0x34a44, 0x34a50,
4596                 0x34a74, 0x34c24,
4597                 0x34d00, 0x34d00,
4598                 0x34d08, 0x34d14,
4599                 0x34d1c, 0x34d20,
4600                 0x34d3c, 0x34d50,
4601                 0x35200, 0x3520c,
4602                 0x35220, 0x35220,
4603                 0x35240, 0x35240,
4604                 0x35600, 0x3560c,
4605                 0x35a00, 0x35a1c,
4606                 0x35e00, 0x35e20,
4607                 0x35e38, 0x35e3c,
4608                 0x35e80, 0x35e80,
4609                 0x35e88, 0x35ea8,
4610                 0x35eb0, 0x35eb4,
4611                 0x35ec8, 0x35ed4,
4612                 0x35fb8, 0x36004,
4613                 0x36200, 0x36200,
4614                 0x36208, 0x36240,
4615                 0x36248, 0x36280,
4616                 0x36288, 0x362c0,
4617                 0x362c8, 0x362fc,
4618                 0x36600, 0x36630,
4619                 0x36a00, 0x36abc,
4620                 0x36b00, 0x36b70,
4621                 0x37000, 0x37048,
4622                 0x37060, 0x3709c,
4623                 0x370f0, 0x37148,
4624                 0x37160, 0x3719c,
4625                 0x371f0, 0x372e4,
4626                 0x372f8, 0x373e4,
4627                 0x373f8, 0x37448,
4628                 0x37460, 0x3749c,
4629                 0x374f0, 0x37548,
4630                 0x37560, 0x3759c,
4631                 0x375f0, 0x376e4,
4632                 0x376f8, 0x377e4,
4633                 0x377f8, 0x377fc,
4634                 0x37814, 0x37814,
4635                 0x3782c, 0x3782c,
4636                 0x37880, 0x3788c,
4637                 0x378e8, 0x378ec,
4638                 0x37900, 0x37948,
4639                 0x37960, 0x3799c,
4640                 0x379f0, 0x37ae4,
4641                 0x37af8, 0x37b10,
4642                 0x37b28, 0x37b28,
4643                 0x37b3c, 0x37b50,
4644                 0x37bf0, 0x37c10,
4645                 0x37c28, 0x37c28,
4646                 0x37c3c, 0x37c50,
4647                 0x37cf0, 0x37cfc,
4648                 0x38000, 0x38030,
4649                 0x38100, 0x38144,
4650                 0x38190, 0x381d0,
4651                 0x38200, 0x38318,
4652                 0x38400, 0x3852c,
4653                 0x38540, 0x3861c,
4654                 0x38800, 0x38834,
4655                 0x388c0, 0x38908,
4656                 0x38910, 0x389ac,
4657                 0x38a00, 0x38a2c,
4658                 0x38a44, 0x38a50,
4659                 0x38a74, 0x38c24,
4660                 0x38d00, 0x38d00,
4661                 0x38d08, 0x38d14,
4662                 0x38d1c, 0x38d20,
4663                 0x38d3c, 0x38d50,
4664                 0x39200, 0x3920c,
4665                 0x39220, 0x39220,
4666                 0x39240, 0x39240,
4667                 0x39600, 0x3960c,
4668                 0x39a00, 0x39a1c,
4669                 0x39e00, 0x39e20,
4670                 0x39e38, 0x39e3c,
4671                 0x39e80, 0x39e80,
4672                 0x39e88, 0x39ea8,
4673                 0x39eb0, 0x39eb4,
4674                 0x39ec8, 0x39ed4,
4675                 0x39fb8, 0x3a004,
4676                 0x3a200, 0x3a200,
4677                 0x3a208, 0x3a240,
4678                 0x3a248, 0x3a280,
4679                 0x3a288, 0x3a2c0,
4680                 0x3a2c8, 0x3a2fc,
4681                 0x3a600, 0x3a630,
4682                 0x3aa00, 0x3aabc,
4683                 0x3ab00, 0x3ab70,
4684                 0x3b000, 0x3b048,
4685                 0x3b060, 0x3b09c,
4686                 0x3b0f0, 0x3b148,
4687                 0x3b160, 0x3b19c,
4688                 0x3b1f0, 0x3b2e4,
4689                 0x3b2f8, 0x3b3e4,
4690                 0x3b3f8, 0x3b448,
4691                 0x3b460, 0x3b49c,
4692                 0x3b4f0, 0x3b548,
4693                 0x3b560, 0x3b59c,
4694                 0x3b5f0, 0x3b6e4,
4695                 0x3b6f8, 0x3b7e4,
4696                 0x3b7f8, 0x3b7fc,
4697                 0x3b814, 0x3b814,
4698                 0x3b82c, 0x3b82c,
4699                 0x3b880, 0x3b88c,
4700                 0x3b8e8, 0x3b8ec,
4701                 0x3b900, 0x3b948,
4702                 0x3b960, 0x3b99c,
4703                 0x3b9f0, 0x3bae4,
4704                 0x3baf8, 0x3bb10,
4705                 0x3bb28, 0x3bb28,
4706                 0x3bb3c, 0x3bb50,
4707                 0x3bbf0, 0x3bc10,
4708                 0x3bc28, 0x3bc28,
4709                 0x3bc3c, 0x3bc50,
4710                 0x3bcf0, 0x3bcfc,
4711                 0x3c000, 0x3c030,
4712                 0x3c100, 0x3c144,
4713                 0x3c190, 0x3c1d0,
4714                 0x3c200, 0x3c318,
4715                 0x3c400, 0x3c52c,
4716                 0x3c540, 0x3c61c,
4717                 0x3c800, 0x3c834,
4718                 0x3c8c0, 0x3c908,
4719                 0x3c910, 0x3c9ac,
4720                 0x3ca00, 0x3ca2c,
4721                 0x3ca44, 0x3ca50,
4722                 0x3ca74, 0x3cc24,
4723                 0x3cd00, 0x3cd00,
4724                 0x3cd08, 0x3cd14,
4725                 0x3cd1c, 0x3cd20,
4726                 0x3cd3c, 0x3cd50,
4727                 0x3d200, 0x3d20c,
4728                 0x3d220, 0x3d220,
4729                 0x3d240, 0x3d240,
4730                 0x3d600, 0x3d60c,
4731                 0x3da00, 0x3da1c,
4732                 0x3de00, 0x3de20,
4733                 0x3de38, 0x3de3c,
4734                 0x3de80, 0x3de80,
4735                 0x3de88, 0x3dea8,
4736                 0x3deb0, 0x3deb4,
4737                 0x3dec8, 0x3ded4,
4738                 0x3dfb8, 0x3e004,
4739                 0x3e200, 0x3e200,
4740                 0x3e208, 0x3e240,
4741                 0x3e248, 0x3e280,
4742                 0x3e288, 0x3e2c0,
4743                 0x3e2c8, 0x3e2fc,
4744                 0x3e600, 0x3e630,
4745                 0x3ea00, 0x3eabc,
4746                 0x3eb00, 0x3eb70,
4747                 0x3f000, 0x3f048,
4748                 0x3f060, 0x3f09c,
4749                 0x3f0f0, 0x3f148,
4750                 0x3f160, 0x3f19c,
4751                 0x3f1f0, 0x3f2e4,
4752                 0x3f2f8, 0x3f3e4,
4753                 0x3f3f8, 0x3f448,
4754                 0x3f460, 0x3f49c,
4755                 0x3f4f0, 0x3f548,
4756                 0x3f560, 0x3f59c,
4757                 0x3f5f0, 0x3f6e4,
4758                 0x3f6f8, 0x3f7e4,
4759                 0x3f7f8, 0x3f7fc,
4760                 0x3f814, 0x3f814,
4761                 0x3f82c, 0x3f82c,
4762                 0x3f880, 0x3f88c,
4763                 0x3f8e8, 0x3f8ec,
4764                 0x3f900, 0x3f948,
4765                 0x3f960, 0x3f99c,
4766                 0x3f9f0, 0x3fae4,
4767                 0x3faf8, 0x3fb10,
4768                 0x3fb28, 0x3fb28,
4769                 0x3fb3c, 0x3fb50,
4770                 0x3fbf0, 0x3fc10,
4771                 0x3fc28, 0x3fc28,
4772                 0x3fc3c, 0x3fc50,
4773                 0x3fcf0, 0x3fcfc,
4774                 0x40000, 0x4000c,
4775                 0x40040, 0x40068,
4776                 0x4007c, 0x40144,
4777                 0x40180, 0x4018c,
4778                 0x40200, 0x40298,
4779                 0x402ac, 0x4033c,
4780                 0x403f8, 0x403fc,
4781                 0x41304, 0x413c4,
4782                 0x41400, 0x4141c,
4783                 0x41480, 0x414d0,
4784                 0x44000, 0x44078,
4785                 0x440c0, 0x44278,
4786                 0x442c0, 0x44478,
4787                 0x444c0, 0x44678,
4788                 0x446c0, 0x44878,
4789                 0x448c0, 0x449fc,
4790                 0x45000, 0x45068,
4791                 0x45080, 0x45084,
4792                 0x450a0, 0x450b0,
4793                 0x45200, 0x45268,
4794                 0x45280, 0x45284,
4795                 0x452a0, 0x452b0,
4796                 0x460c0, 0x460e4,
4797                 0x47000, 0x4708c,
4798                 0x47200, 0x47250,
4799                 0x47400, 0x47420,
4800                 0x47600, 0x47618,
4801                 0x47800, 0x47814,
4802                 0x48000, 0x4800c,
4803                 0x48040, 0x48068,
4804                 0x4807c, 0x48144,
4805                 0x48180, 0x4818c,
4806                 0x48200, 0x48298,
4807                 0x482ac, 0x4833c,
4808                 0x483f8, 0x483fc,
4809                 0x49304, 0x493c4,
4810                 0x49400, 0x4941c,
4811                 0x49480, 0x494d0,
4812                 0x4c000, 0x4c078,
4813                 0x4c0c0, 0x4c278,
4814                 0x4c2c0, 0x4c478,
4815                 0x4c4c0, 0x4c678,
4816                 0x4c6c0, 0x4c878,
4817                 0x4c8c0, 0x4c9fc,
4818                 0x4d000, 0x4d068,
4819                 0x4d080, 0x4d084,
4820                 0x4d0a0, 0x4d0b0,
4821                 0x4d200, 0x4d268,
4822                 0x4d280, 0x4d284,
4823                 0x4d2a0, 0x4d2b0,
4824                 0x4e0c0, 0x4e0e4,
4825                 0x4f000, 0x4f08c,
4826                 0x4f200, 0x4f250,
4827                 0x4f400, 0x4f420,
4828                 0x4f600, 0x4f618,
4829                 0x4f800, 0x4f814,
4830                 0x50000, 0x500cc,
4831                 0x50400, 0x50400,
4832                 0x50800, 0x508cc,
4833                 0x50c00, 0x50c00,
4834                 0x51000, 0x5101c,
4835                 0x51300, 0x51308,
4836         };
4837
4838         if (is_t4(sc)) {
4839                 reg_ranges = &t4_reg_ranges[0];
4840                 n = nitems(t4_reg_ranges);
4841         } else {
4842                 reg_ranges = &t5_reg_ranges[0];
4843                 n = nitems(t5_reg_ranges);
4844         }
4845
4846         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4847         for (i = 0; i < n; i += 2)
4848                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4849 }
4850
4851 #define A_PL_INDIR_CMD  0x1f8
4852
4853 #define S_PL_AUTOINC    31
4854 #define M_PL_AUTOINC    0x1U
4855 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
4856 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
4857
4858 #define S_PL_VFID       20
4859 #define M_PL_VFID       0xffU
4860 #define V_PL_VFID(x)    ((x) << S_PL_VFID)
4861 #define G_PL_VFID(x)    (((x) >> S_PL_VFID) & M_PL_VFID)
4862
4863 #define S_PL_ADDR       0
4864 #define M_PL_ADDR       0xfffffU
4865 #define V_PL_ADDR(x)    ((x) << S_PL_ADDR)
4866 #define G_PL_ADDR(x)    (((x) >> S_PL_ADDR) & M_PL_ADDR)
4867
4868 #define A_PL_INDIR_DATA 0x1fc
4869
4870 static uint64_t
4871 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
4872 {
4873         u32 stats[2];
4874
4875         mtx_assert(&sc->regwin_lock, MA_OWNED);
4876         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4877             V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg)));
4878         stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
4879         stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
4880         return (((uint64_t)stats[1]) << 32 | stats[0]);
4881 }
4882
4883 static void
4884 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
4885     struct fw_vi_stats_vf *stats)
4886 {
4887
4888 #define GET_STAT(name) \
4889         read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
4890
4891         stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
4892         stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
4893         stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
4894         stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
4895         stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
4896         stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
4897         stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
4898         stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
4899         stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
4900         stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
4901         stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
4902         stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
4903         stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
4904         stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
4905         stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
4906         stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
4907
4908 #undef GET_STAT
4909 }
4910
4911 static void
4912 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
4913 {
4914         int reg;
4915
4916         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4917             V_PL_VFID(G_FW_VIID_VIN(viid)) |
4918             V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
4919         for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
4920              reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
4921                 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
4922 }
4923
4924 static void
4925 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
4926 {
4927         struct timeval tv;
4928         const struct timeval interval = {0, 250000};    /* 250ms */
4929
4930         if (!(vi->flags & VI_INIT_DONE))
4931                 return;
4932
4933         getmicrotime(&tv);
4934         timevalsub(&tv, &interval);
4935         if (timevalcmp(&tv, &vi->last_refreshed, <))
4936                 return;
4937
4938         mtx_lock(&sc->regwin_lock);
4939         t4_get_vi_stats(sc, vi->viid, &vi->stats);
4940         getmicrotime(&vi->last_refreshed);
4941         mtx_unlock(&sc->regwin_lock);
4942 }
4943
4944 static void
4945 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4946 {
4947         int i;
4948         u_int v, tnl_cong_drops;
4949         struct timeval tv;
4950         const struct timeval interval = {0, 250000};    /* 250ms */
4951
4952         getmicrotime(&tv);
4953         timevalsub(&tv, &interval);
4954         if (timevalcmp(&tv, &pi->last_refreshed, <))
4955                 return;
4956
4957         tnl_cong_drops = 0;
4958         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4959         for (i = 0; i < NCHAN; i++) {
4960                 if (pi->rx_chan_map & (1 << i)) {
4961                         mtx_lock(&sc->regwin_lock);
4962                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4963                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4964                         mtx_unlock(&sc->regwin_lock);
4965                         tnl_cong_drops += v;
4966                 }
4967         }
4968         pi->tnl_cong_drops = tnl_cong_drops;
4969         getmicrotime(&pi->last_refreshed);
4970 }
4971
4972 static void
4973 cxgbe_tick(void *arg)
4974 {
4975         struct port_info *pi = arg;
4976         struct adapter *sc = pi->adapter;
4977
4978         PORT_LOCK_ASSERT_OWNED(pi);
4979         cxgbe_refresh_stats(sc, pi);
4980
4981         callout_schedule(&pi->tick, hz);
4982 }
4983
4984 void
4985 vi_tick(void *arg)
4986 {
4987         struct vi_info *vi = arg;
4988         struct adapter *sc = vi->pi->adapter;
4989
4990         vi_refresh_stats(sc, vi);
4991
4992         callout_schedule(&vi->tick, hz);
4993 }
4994
4995 static void
4996 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4997 {
4998         struct ifnet *vlan;
4999
5000         if (arg != ifp || ifp->if_type != IFT_ETHER)
5001                 return;
5002
5003         vlan = VLAN_DEVAT(ifp, vid);
5004         VLAN_SETCOOKIE(vlan, ifp);
5005 }
5006
5007 static int
5008 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5009 {
5010
5011 #ifdef INVARIANTS
5012         panic("%s: opcode 0x%02x on iq %p with payload %p",
5013             __func__, rss->opcode, iq, m);
5014 #else
5015         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
5016             __func__, rss->opcode, iq, m);
5017         m_freem(m);
5018 #endif
5019         return (EDOOFUS);
5020 }
5021
5022 int
5023 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
5024 {
5025         uintptr_t *loc, new;
5026
5027         if (opcode >= nitems(sc->cpl_handler))
5028                 return (EINVAL);
5029
5030         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
5031         loc = (uintptr_t *) &sc->cpl_handler[opcode];
5032         atomic_store_rel_ptr(loc, new);
5033
5034         return (0);
5035 }
5036
5037 static int
5038 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
5039 {
5040
5041 #ifdef INVARIANTS
5042         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
5043 #else
5044         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
5045             __func__, iq, ctrl);
5046 #endif
5047         return (EDOOFUS);
5048 }
5049
5050 int
5051 t4_register_an_handler(struct adapter *sc, an_handler_t h)
5052 {
5053         uintptr_t *loc, new;
5054
5055         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
5056         loc = (uintptr_t *) &sc->an_handler;
5057         atomic_store_rel_ptr(loc, new);
5058
5059         return (0);
5060 }
5061
5062 static int
5063 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
5064 {
5065         const struct cpl_fw6_msg *cpl =
5066             __containerof(rpl, struct cpl_fw6_msg, data[0]);
5067
5068 #ifdef INVARIANTS
5069         panic("%s: fw_msg type %d", __func__, cpl->type);
5070 #else
5071         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
5072 #endif
5073         return (EDOOFUS);
5074 }
5075
5076 int
5077 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
5078 {
5079         uintptr_t *loc, new;
5080
5081         if (type >= nitems(sc->fw_msg_handler))
5082                 return (EINVAL);
5083
5084         /*
5085          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
5086          * handler dispatch table.  Reject any attempt to install a handler for
5087          * this subtype.
5088          */
5089         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
5090                 return (EINVAL);
5091
5092         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
5093         loc = (uintptr_t *) &sc->fw_msg_handler[type];
5094         atomic_store_rel_ptr(loc, new);
5095
5096         return (0);
5097 }
5098
5099 static void
5100 t4_sysctls(struct adapter *sc)
5101 {
5102         struct sysctl_ctx_list *ctx;
5103         struct sysctl_oid *oid;
5104         struct sysctl_oid_list *children, *c0;
5105         static char *caps[] = {
5106                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
5107                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
5108                     "\6HASHFILTER\7ETHOFLD",
5109                 "\20\1TOE",                             /* caps[2] toecaps */
5110                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
5111                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
5112                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
5113                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
5114                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
5115                     "\4PO_INITIAOR\5PO_TARGET"
5116         };
5117         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
5118
5119         ctx = device_get_sysctl_ctx(sc->dev);
5120
5121         /*
5122          * dev.t4nex.X.
5123          */
5124         oid = device_get_sysctl_tree(sc->dev);
5125         c0 = children = SYSCTL_CHILDREN(oid);
5126
5127         sc->sc_do_rxcopy = 1;
5128         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
5129             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
5130
5131         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
5132             sc->params.nports, "# of ports");
5133
5134         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
5135             NULL, chip_rev(sc), "chip hardware revision");
5136
5137         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
5138             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
5139
5140         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
5141             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
5142
5143         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
5144             sc->cfcsum, "config file checksum");
5145
5146         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
5147             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
5148             sysctl_bitfield, "A", "available doorbells");
5149
5150         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
5151             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
5152             sysctl_bitfield, "A", "available link capabilities");
5153
5154         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
5155             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
5156             sysctl_bitfield, "A", "available NIC capabilities");
5157
5158         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
5159             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
5160             sysctl_bitfield, "A", "available TCP offload capabilities");
5161
5162         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
5163             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
5164             sysctl_bitfield, "A", "available RDMA capabilities");
5165
5166         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
5167             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
5168             sysctl_bitfield, "A", "available iSCSI capabilities");
5169
5170         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
5171             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
5172             sysctl_bitfield, "A", "available FCoE capabilities");
5173
5174         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
5175             sc->params.vpd.cclk, "core clock frequency (in KHz)");
5176
5177         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
5178             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
5179             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
5180             "interrupt holdoff timer values (us)");
5181
5182         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
5183             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
5184             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
5185             "interrupt holdoff packet counter values");
5186
5187         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
5188             NULL, sc->tids.nftids, "number of filters");
5189
5190         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
5191             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
5192             "chip temperature (in Celsius)");
5193
5194         t4_sge_sysctls(sc, ctx, children);
5195
5196         sc->lro_timeout = 100;
5197         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
5198             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
5199
5200         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
5201             &sc->debug_flags, 0, "flags to enable runtime debugging");
5202
5203 #ifdef SBUF_DRAIN
5204         /*
5205          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
5206          */
5207         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
5208             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
5209             "logs and miscellaneous information");
5210         children = SYSCTL_CHILDREN(oid);
5211
5212         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
5213             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5214             sysctl_cctrl, "A", "congestion control");
5215
5216         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
5217             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5218             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
5219
5220         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
5221             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
5222             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
5223
5224         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
5225             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
5226             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
5227
5228         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
5229             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
5230             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
5231
5232         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
5233             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
5234             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
5235
5236         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
5237             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
5238             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
5239
5240         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
5241             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5242             sysctl_cim_la, "A", "CIM logic analyzer");
5243
5244         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
5245             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5246             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
5247
5248         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
5249             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
5250             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
5251
5252         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
5253             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
5254             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
5255
5256         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
5257             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
5258             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
5259
5260         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
5261             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
5262             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
5263
5264         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
5265             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
5266             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
5267
5268         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
5269             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
5270             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
5271
5272         if (is_t5(sc)) {
5273                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
5274                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
5275                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
5276
5277                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
5278                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
5279                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
5280         }
5281
5282         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
5283             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5284             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
5285
5286         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
5287             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5288             sysctl_cim_qcfg, "A", "CIM queue configuration");
5289
5290         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5291             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5292             sysctl_cpl_stats, "A", "CPL statistics");
5293
5294         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5295             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5296             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5297
5298         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5299             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5300             sysctl_devlog, "A", "firmware's device log");
5301
5302         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5303             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5304             sysctl_fcoe_stats, "A", "FCoE statistics");
5305
5306         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5307             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5308             sysctl_hw_sched, "A", "hardware scheduler ");
5309
5310         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5311             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5312             sysctl_l2t, "A", "hardware L2 table");
5313
5314         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5315             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5316             sysctl_lb_stats, "A", "loopback statistics");
5317
5318         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5319             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5320             sysctl_meminfo, "A", "memory regions");
5321
5322         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5323             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5324             sysctl_mps_tcam, "A", "MPS TCAM entries");
5325
5326         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5327             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5328             sysctl_path_mtus, "A", "path MTUs");
5329
5330         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5331             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5332             sysctl_pm_stats, "A", "PM statistics");
5333
5334         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5335             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5336             sysctl_rdma_stats, "A", "RDMA statistics");
5337
5338         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5339             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5340             sysctl_tcp_stats, "A", "TCP statistics");
5341
5342         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5343             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5344             sysctl_tids, "A", "TID information");
5345
5346         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5347             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5348             sysctl_tp_err_stats, "A", "TP error statistics");
5349
5350         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5351             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5352             sysctl_tp_la, "A", "TP logic analyzer");
5353
5354         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5355             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5356             sysctl_tx_rate, "A", "Tx rate");
5357
5358         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5359             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5360             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5361
5362         if (is_t5(sc)) {
5363                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5364                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5365                     sysctl_wcwr_stats, "A", "write combined work requests");
5366         }
5367 #endif
5368
5369 #ifdef TCP_OFFLOAD
5370         if (is_offload(sc)) {
5371                 /*
5372                  * dev.t4nex.X.toe.
5373                  */
5374                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5375                     NULL, "TOE parameters");
5376                 children = SYSCTL_CHILDREN(oid);
5377
5378                 sc->tt.sndbuf = 256 * 1024;
5379                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5380                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
5381
5382                 sc->tt.ddp = 0;
5383                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5384                     &sc->tt.ddp, 0, "DDP allowed");
5385
5386                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
5387                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
5388                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
5389
5390                 sc->tt.ddp_thres =
5391                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
5392                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
5393                     &sc->tt.ddp_thres, 0, "DDP threshold");
5394
5395                 sc->tt.rx_coalesce = 1;
5396                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5397                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5398
5399                 sc->tt.tx_align = 1;
5400                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5401                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5402         }
5403 #endif
5404 }
5405
5406 void
5407 vi_sysctls(struct vi_info *vi)
5408 {
5409         struct sysctl_ctx_list *ctx;
5410         struct sysctl_oid *oid;
5411         struct sysctl_oid_list *children;
5412
5413         ctx = device_get_sysctl_ctx(vi->dev);
5414
5415         /*
5416          * dev.[nv](cxgbe|cxl).X.
5417          */
5418         oid = device_get_sysctl_tree(vi->dev);
5419         children = SYSCTL_CHILDREN(oid);
5420
5421         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
5422             vi->viid, "VI identifer");
5423         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5424             &vi->nrxq, 0, "# of rx queues");
5425         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5426             &vi->ntxq, 0, "# of tx queues");
5427         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5428             &vi->first_rxq, 0, "index of first rx queue");
5429         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5430             &vi->first_txq, 0, "index of first tx queue");
5431
5432         if (vi->flags & VI_NETMAP)
5433                 return;
5434
5435         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
5436             CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
5437             "Reserve queue 0 for non-flowid packets");
5438
5439 #ifdef TCP_OFFLOAD
5440         if (vi->nofldrxq != 0) {
5441                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5442                     &vi->nofldrxq, 0,
5443                     "# of rx queues for offloaded TCP connections");
5444                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5445                     &vi->nofldtxq, 0,
5446                     "# of tx queues for offloaded TCP connections");
5447                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5448                     CTLFLAG_RD, &vi->first_ofld_rxq, 0,
5449                     "index of first TOE rx queue");
5450                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5451                     CTLFLAG_RD, &vi->first_ofld_txq, 0,
5452                     "index of first TOE tx queue");
5453         }
5454 #endif
5455
5456         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5457             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
5458             "holdoff timer index");
5459         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5460             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
5461             "holdoff packet counter index");
5462
5463         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5464             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
5465             "rx queue size");
5466         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5467             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
5468             "tx queue size");
5469 }
5470
5471 static void
5472 cxgbe_sysctls(struct port_info *pi)
5473 {
5474         struct sysctl_ctx_list *ctx;
5475         struct sysctl_oid *oid;
5476         struct sysctl_oid_list *children;
5477         struct adapter *sc = pi->adapter;
5478
5479         ctx = device_get_sysctl_ctx(pi->dev);
5480
5481         /*
5482          * dev.cxgbe.X.
5483          */
5484         oid = device_get_sysctl_tree(pi->dev);
5485         children = SYSCTL_CHILDREN(oid);
5486
5487         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5488            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5489         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5490                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5491                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5492                     "PHY temperature (in Celsius)");
5493                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5494                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5495                     "PHY firmware version");
5496         }
5497
5498         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5499             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
5500             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5501
5502         /*
5503          * dev.cxgbe.X.stats.
5504          */
5505         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5506             NULL, "port statistics");
5507         children = SYSCTL_CHILDREN(oid);
5508         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5509             &pi->tx_parse_error, 0,
5510             "# of tx packets with invalid length or # of segments");
5511
5512 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5513         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5514             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5515             sysctl_handle_t4_reg64, "QU", desc)
5516
5517         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5518             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5519         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5520             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5521         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5522             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5523         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5524             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5525         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5526             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5527         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5528             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5529         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5530             "# of tx frames in this range",
5531             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5532         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5533             "# of tx frames in this range",
5534             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5535         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5536             "# of tx frames in this range",
5537             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5538         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5539             "# of tx frames in this range",
5540             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5541         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5542             "# of tx frames in this range",
5543             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5544         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5545             "# of tx frames in this range",
5546             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5547         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5548             "# of tx frames in this range",
5549             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5550         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5551             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5552         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5553             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5554         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5555             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5556         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5557             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5558         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5559             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5560         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5561             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5562         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5563             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5564         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5565             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5566         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5567             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5568         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5569             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5570
5571         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5572             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5573         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5574             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5575         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5576             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5577         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5578             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5579         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5580             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5581         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5582             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5583         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5584             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5585         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5586             "# of frames received with bad FCS",
5587             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5588         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5589             "# of frames received with length error",
5590             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5591         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5592             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5593         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5594             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5595         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5596             "# of rx frames in this range",
5597             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5598         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5599             "# of rx frames in this range",
5600             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5601         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5602             "# of rx frames in this range",
5603             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5604         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5605             "# of rx frames in this range",
5606             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5607         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5608             "# of rx frames in this range",
5609             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5610         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5611             "# of rx frames in this range",
5612             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5613         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5614             "# of rx frames in this range",
5615             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5616         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5617             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5618         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5619             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5620         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5621             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5622         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5623             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5624         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5625             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5626         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5627             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5628         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5629             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5630         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5631             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5632         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5633             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5634
5635 #undef SYSCTL_ADD_T4_REG64
5636
5637 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5638         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5639             &pi->stats.name, desc)
5640
5641         /* We get these from port_stats and they may be stale by upto 1s */
5642         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5643             "# drops due to buffer-group 0 overflows");
5644         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5645             "# drops due to buffer-group 1 overflows");
5646         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5647             "# drops due to buffer-group 2 overflows");
5648         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5649             "# drops due to buffer-group 3 overflows");
5650         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5651             "# of buffer-group 0 truncated packets");
5652         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5653             "# of buffer-group 1 truncated packets");
5654         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5655             "# of buffer-group 2 truncated packets");
5656         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5657             "# of buffer-group 3 truncated packets");
5658
5659 #undef SYSCTL_ADD_T4_PORTSTAT
5660 }
5661
5662 static int
5663 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5664 {
5665         int rc, *i, space = 0;
5666         struct sbuf sb;
5667
5668         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5669         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5670                 if (space)
5671                         sbuf_printf(&sb, " ");
5672                 sbuf_printf(&sb, "%d", *i);
5673                 space = 1;
5674         }
5675         rc = sbuf_finish(&sb);
5676         sbuf_delete(&sb);
5677         return (rc);
5678 }
5679
5680 static int
5681 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5682 {
5683         int rc;
5684         struct sbuf *sb;
5685
5686         rc = sysctl_wire_old_buffer(req, 0);
5687         if (rc != 0)
5688                 return(rc);
5689
5690         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5691         if (sb == NULL)
5692                 return (ENOMEM);
5693
5694         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5695         rc = sbuf_finish(sb);
5696         sbuf_delete(sb);
5697
5698         return (rc);
5699 }
5700
5701 static int
5702 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5703 {
5704         struct port_info *pi = arg1;
5705         int op = arg2;
5706         struct adapter *sc = pi->adapter;
5707         u_int v;
5708         int rc;
5709
5710         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
5711         if (rc)
5712                 return (rc);
5713         /* XXX: magic numbers */
5714         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5715             &v);
5716         end_synchronized_op(sc, 0);
5717         if (rc)
5718                 return (rc);
5719         if (op == 0)
5720                 v /= 256;
5721
5722         rc = sysctl_handle_int(oidp, &v, 0, req);
5723         return (rc);
5724 }
5725
5726 static int
5727 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5728 {
5729         struct vi_info *vi = arg1;
5730         int rc, val;
5731
5732         val = vi->rsrv_noflowq;
5733         rc = sysctl_handle_int(oidp, &val, 0, req);
5734         if (rc != 0 || req->newptr == NULL)
5735                 return (rc);
5736
5737         if ((val >= 1) && (vi->ntxq > 1))
5738                 vi->rsrv_noflowq = 1;
5739         else
5740                 vi->rsrv_noflowq = 0;
5741
5742         return (rc);
5743 }
5744
5745 static int
5746 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5747 {
5748         struct vi_info *vi = arg1;
5749         struct adapter *sc = vi->pi->adapter;
5750         int idx, rc, i;
5751         struct sge_rxq *rxq;
5752 #ifdef TCP_OFFLOAD
5753         struct sge_ofld_rxq *ofld_rxq;
5754 #endif
5755         uint8_t v;
5756
5757         idx = vi->tmr_idx;
5758
5759         rc = sysctl_handle_int(oidp, &idx, 0, req);
5760         if (rc != 0 || req->newptr == NULL)
5761                 return (rc);
5762
5763         if (idx < 0 || idx >= SGE_NTIMERS)
5764                 return (EINVAL);
5765
5766         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5767             "t4tmr");
5768         if (rc)
5769                 return (rc);
5770
5771         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
5772         for_each_rxq(vi, i, rxq) {
5773 #ifdef atomic_store_rel_8
5774                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5775 #else
5776                 rxq->iq.intr_params = v;
5777 #endif
5778         }
5779 #ifdef TCP_OFFLOAD
5780         for_each_ofld_rxq(vi, i, ofld_rxq) {
5781 #ifdef atomic_store_rel_8
5782                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5783 #else
5784                 ofld_rxq->iq.intr_params = v;
5785 #endif
5786         }
5787 #endif
5788         vi->tmr_idx = idx;
5789
5790         end_synchronized_op(sc, LOCK_HELD);
5791         return (0);
5792 }
5793
5794 static int
5795 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5796 {
5797         struct vi_info *vi = arg1;
5798         struct adapter *sc = vi->pi->adapter;
5799         int idx, rc;
5800
5801         idx = vi->pktc_idx;
5802
5803         rc = sysctl_handle_int(oidp, &idx, 0, req);
5804         if (rc != 0 || req->newptr == NULL)
5805                 return (rc);
5806
5807         if (idx < -1 || idx >= SGE_NCOUNTERS)
5808                 return (EINVAL);
5809
5810         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5811             "t4pktc");
5812         if (rc)
5813                 return (rc);
5814
5815         if (vi->flags & VI_INIT_DONE)
5816                 rc = EBUSY; /* cannot be changed once the queues are created */
5817         else
5818                 vi->pktc_idx = idx;
5819
5820         end_synchronized_op(sc, LOCK_HELD);
5821         return (rc);
5822 }
5823
5824 static int
5825 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5826 {
5827         struct vi_info *vi = arg1;
5828         struct adapter *sc = vi->pi->adapter;
5829         int qsize, rc;
5830
5831         qsize = vi->qsize_rxq;
5832
5833         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5834         if (rc != 0 || req->newptr == NULL)
5835                 return (rc);
5836
5837         if (qsize < 128 || (qsize & 7))
5838                 return (EINVAL);
5839
5840         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5841             "t4rxqs");
5842         if (rc)
5843                 return (rc);
5844
5845         if (vi->flags & VI_INIT_DONE)
5846                 rc = EBUSY; /* cannot be changed once the queues are created */
5847         else
5848                 vi->qsize_rxq = qsize;
5849
5850         end_synchronized_op(sc, LOCK_HELD);
5851         return (rc);
5852 }
5853
5854 static int
5855 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5856 {
5857         struct vi_info *vi = arg1;
5858         struct adapter *sc = vi->pi->adapter;
5859         int qsize, rc;
5860
5861         qsize = vi->qsize_txq;
5862
5863         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5864         if (rc != 0 || req->newptr == NULL)
5865                 return (rc);
5866
5867         if (qsize < 128 || qsize > 65536)
5868                 return (EINVAL);
5869
5870         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5871             "t4txqs");
5872         if (rc)
5873                 return (rc);
5874
5875         if (vi->flags & VI_INIT_DONE)
5876                 rc = EBUSY; /* cannot be changed once the queues are created */
5877         else
5878                 vi->qsize_txq = qsize;
5879
5880         end_synchronized_op(sc, LOCK_HELD);
5881         return (rc);
5882 }
5883
5884 static int
5885 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5886 {
5887         struct port_info *pi = arg1;
5888         struct adapter *sc = pi->adapter;
5889         struct link_config *lc = &pi->link_cfg;
5890         int rc;
5891
5892         if (req->newptr == NULL) {
5893                 struct sbuf *sb;
5894                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5895
5896                 rc = sysctl_wire_old_buffer(req, 0);
5897                 if (rc != 0)
5898                         return(rc);
5899
5900                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5901                 if (sb == NULL)
5902                         return (ENOMEM);
5903
5904                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5905                 rc = sbuf_finish(sb);
5906                 sbuf_delete(sb);
5907         } else {
5908                 char s[2];
5909                 int n;
5910
5911                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5912                 s[1] = 0;
5913
5914                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5915                 if (rc != 0)
5916                         return(rc);
5917
5918                 if (s[1] != 0)
5919                         return (EINVAL);
5920                 if (s[0] < '0' || s[0] > '9')
5921                         return (EINVAL);        /* not a number */
5922                 n = s[0] - '0';
5923                 if (n & ~(PAUSE_TX | PAUSE_RX))
5924                         return (EINVAL);        /* some other bit is set too */
5925
5926                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5927                     "t4PAUSE");
5928                 if (rc)
5929                         return (rc);
5930                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5931                         int link_ok = lc->link_ok;
5932
5933                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5934                         lc->requested_fc |= n;
5935                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5936                         lc->link_ok = link_ok;  /* restore */
5937                 }
5938                 end_synchronized_op(sc, 0);
5939         }
5940
5941         return (rc);
5942 }
5943
5944 static int
5945 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5946 {
5947         struct adapter *sc = arg1;
5948         int reg = arg2;
5949         uint64_t val;
5950
5951         val = t4_read_reg64(sc, reg);
5952
5953         return (sysctl_handle_64(oidp, &val, 0, req));
5954 }
5955
5956 static int
5957 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5958 {
5959         struct adapter *sc = arg1;
5960         int rc, t;
5961         uint32_t param, val;
5962
5963         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5964         if (rc)
5965                 return (rc);
5966         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5967             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5968             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5969         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5970         end_synchronized_op(sc, 0);
5971         if (rc)
5972                 return (rc);
5973
5974         /* unknown is returned as 0 but we display -1 in that case */
5975         t = val == 0 ? -1 : val;
5976
5977         rc = sysctl_handle_int(oidp, &t, 0, req);
5978         return (rc);
5979 }
5980
5981 #ifdef SBUF_DRAIN
5982 static int
5983 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5984 {
5985         struct adapter *sc = arg1;
5986         struct sbuf *sb;
5987         int rc, i;
5988         uint16_t incr[NMTUS][NCCTRL_WIN];
5989         static const char *dec_fac[] = {
5990                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5991                 "0.9375"
5992         };
5993
5994         rc = sysctl_wire_old_buffer(req, 0);
5995         if (rc != 0)
5996                 return (rc);
5997
5998         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5999         if (sb == NULL)
6000                 return (ENOMEM);
6001
6002         t4_read_cong_tbl(sc, incr);
6003
6004         for (i = 0; i < NCCTRL_WIN; ++i) {
6005                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
6006                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
6007                     incr[5][i], incr[6][i], incr[7][i]);
6008                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
6009                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
6010                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
6011                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
6012         }
6013
6014         rc = sbuf_finish(sb);
6015         sbuf_delete(sb);
6016
6017         return (rc);
6018 }
6019
6020 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
6021         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
6022         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
6023         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
6024 };
6025
6026 static int
6027 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
6028 {
6029         struct adapter *sc = arg1;
6030         struct sbuf *sb;
6031         int rc, i, n, qid = arg2;
6032         uint32_t *buf, *p;
6033         char *qtype;
6034         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
6035
6036         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
6037             ("%s: bad qid %d\n", __func__, qid));
6038
6039         if (qid < CIM_NUM_IBQ) {
6040                 /* inbound queue */
6041                 qtype = "IBQ";
6042                 n = 4 * CIM_IBQ_SIZE;
6043                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
6044                 rc = t4_read_cim_ibq(sc, qid, buf, n);
6045         } else {
6046                 /* outbound queue */
6047                 qtype = "OBQ";
6048                 qid -= CIM_NUM_IBQ;
6049                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
6050                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
6051                 rc = t4_read_cim_obq(sc, qid, buf, n);
6052         }
6053
6054         if (rc < 0) {
6055                 rc = -rc;
6056                 goto done;
6057         }
6058         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
6059
6060         rc = sysctl_wire_old_buffer(req, 0);
6061         if (rc != 0)
6062                 goto done;
6063
6064         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6065         if (sb == NULL) {
6066                 rc = ENOMEM;
6067                 goto done;
6068         }
6069
6070         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
6071         for (i = 0, p = buf; i < n; i += 16, p += 4)
6072                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
6073                     p[2], p[3]);
6074
6075         rc = sbuf_finish(sb);
6076         sbuf_delete(sb);
6077 done:
6078         free(buf, M_CXGBE);
6079         return (rc);
6080 }
6081
6082 static int
6083 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
6084 {
6085         struct adapter *sc = arg1;
6086         u_int cfg;
6087         struct sbuf *sb;
6088         uint32_t *buf, *p;
6089         int rc;
6090
6091         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
6092         if (rc != 0)
6093                 return (rc);
6094
6095         rc = sysctl_wire_old_buffer(req, 0);
6096         if (rc != 0)
6097                 return (rc);
6098
6099         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6100         if (sb == NULL)
6101                 return (ENOMEM);
6102
6103         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
6104             M_ZERO | M_WAITOK);
6105
6106         rc = -t4_cim_read_la(sc, buf, NULL);
6107         if (rc != 0)
6108                 goto done;
6109
6110         sbuf_printf(sb, "Status   Data      PC%s",
6111             cfg & F_UPDBGLACAPTPCONLY ? "" :
6112             "     LS0Stat  LS0Addr             LS0Data");
6113
6114         KASSERT((sc->params.cim_la_size & 7) == 0,
6115             ("%s: p will walk off the end of buf", __func__));
6116
6117         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
6118                 if (cfg & F_UPDBGLACAPTPCONLY) {
6119                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
6120                             p[6], p[7]);
6121                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
6122                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
6123                             p[4] & 0xff, p[5] >> 8);
6124                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
6125                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6126                             p[1] & 0xf, p[2] >> 4);
6127                 } else {
6128                         sbuf_printf(sb,
6129                             "\n  %02x   %x%07x %x%07x %08x %08x "
6130                             "%08x%08x%08x%08x",
6131                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6132                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
6133                             p[6], p[7]);
6134                 }
6135         }
6136
6137         rc = sbuf_finish(sb);
6138         sbuf_delete(sb);
6139 done:
6140         free(buf, M_CXGBE);
6141         return (rc);
6142 }
6143
6144 static int
6145 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
6146 {
6147         struct adapter *sc = arg1;
6148         u_int i;
6149         struct sbuf *sb;
6150         uint32_t *buf, *p;
6151         int rc;
6152
6153         rc = sysctl_wire_old_buffer(req, 0);
6154         if (rc != 0)
6155                 return (rc);
6156
6157         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6158         if (sb == NULL)
6159                 return (ENOMEM);
6160
6161         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
6162             M_ZERO | M_WAITOK);
6163
6164         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
6165         p = buf;
6166
6167         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6168                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
6169                     p[1], p[0]);
6170         }
6171
6172         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
6173         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6174                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
6175                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
6176                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
6177                     (p[1] >> 2) | ((p[2] & 3) << 30),
6178                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
6179                     p[0] & 1);
6180         }
6181
6182         rc = sbuf_finish(sb);
6183         sbuf_delete(sb);
6184         free(buf, M_CXGBE);
6185         return (rc);
6186 }
6187
6188 static int
6189 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
6190 {
6191         struct adapter *sc = arg1;
6192         u_int i;
6193         struct sbuf *sb;
6194         uint32_t *buf, *p;
6195         int rc;
6196
6197         rc = sysctl_wire_old_buffer(req, 0);
6198         if (rc != 0)
6199                 return (rc);
6200
6201         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6202         if (sb == NULL)
6203                 return (ENOMEM);
6204
6205         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
6206             M_ZERO | M_WAITOK);
6207
6208         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
6209         p = buf;
6210
6211         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
6212         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6213                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
6214                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
6215                     p[4], p[3], p[2], p[1], p[0]);
6216         }
6217
6218         sbuf_printf(sb, "\n\nCntl ID               Data");
6219         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6220                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
6221                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
6222         }
6223
6224         rc = sbuf_finish(sb);
6225         sbuf_delete(sb);
6226         free(buf, M_CXGBE);
6227         return (rc);
6228 }
6229
6230 static int
6231 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
6232 {
6233         struct adapter *sc = arg1;
6234         struct sbuf *sb;
6235         int rc, i;
6236         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6237         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6238         uint16_t thres[CIM_NUM_IBQ];
6239         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
6240         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
6241         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
6242
6243         if (is_t4(sc)) {
6244                 cim_num_obq = CIM_NUM_OBQ;
6245                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
6246                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
6247         } else {
6248                 cim_num_obq = CIM_NUM_OBQ_T5;
6249                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
6250                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
6251         }
6252         nq = CIM_NUM_IBQ + cim_num_obq;
6253
6254         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
6255         if (rc == 0)
6256                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
6257         if (rc != 0)
6258                 return (rc);
6259
6260         t4_read_cimq_cfg(sc, base, size, thres);
6261
6262         rc = sysctl_wire_old_buffer(req, 0);
6263         if (rc != 0)
6264                 return (rc);
6265
6266         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6267         if (sb == NULL)
6268                 return (ENOMEM);
6269
6270         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
6271
6272         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
6273                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
6274                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
6275                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6276                     G_QUEREMFLITS(p[2]) * 16);
6277         for ( ; i < nq; i++, p += 4, wr += 2)
6278                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
6279                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
6280                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6281                     G_QUEREMFLITS(p[2]) * 16);
6282
6283         rc = sbuf_finish(sb);
6284         sbuf_delete(sb);
6285
6286         return (rc);
6287 }
6288
6289 static int
6290 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
6291 {
6292         struct adapter *sc = arg1;
6293         struct sbuf *sb;
6294         int rc;
6295         struct tp_cpl_stats stats;
6296
6297         rc = sysctl_wire_old_buffer(req, 0);
6298         if (rc != 0)
6299                 return (rc);
6300
6301         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6302         if (sb == NULL)
6303                 return (ENOMEM);
6304
6305         t4_tp_get_cpl_stats(sc, &stats);
6306
6307         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6308             "channel 3\n");
6309         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
6310                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
6311         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
6312                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
6313
6314         rc = sbuf_finish(sb);
6315         sbuf_delete(sb);
6316
6317         return (rc);
6318 }
6319
6320 static int
6321 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
6322 {
6323         struct adapter *sc = arg1;
6324         struct sbuf *sb;
6325         int rc;
6326         struct tp_usm_stats stats;
6327
6328         rc = sysctl_wire_old_buffer(req, 0);
6329         if (rc != 0)
6330                 return(rc);
6331
6332         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6333         if (sb == NULL)
6334                 return (ENOMEM);
6335
6336         t4_get_usm_stats(sc, &stats);
6337
6338         sbuf_printf(sb, "Frames: %u\n", stats.frames);
6339         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
6340         sbuf_printf(sb, "Drops:  %u", stats.drops);
6341
6342         rc = sbuf_finish(sb);
6343         sbuf_delete(sb);
6344
6345         return (rc);
6346 }
6347
6348 const char *devlog_level_strings[] = {
6349         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
6350         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
6351         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
6352         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
6353         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
6354         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
6355 };
6356
6357 const char *devlog_facility_strings[] = {
6358         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
6359         [FW_DEVLOG_FACILITY_CF]         = "CF",
6360         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
6361         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
6362         [FW_DEVLOG_FACILITY_RES]        = "RES",
6363         [FW_DEVLOG_FACILITY_HW]         = "HW",
6364         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
6365         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
6366         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
6367         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
6368         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
6369         [FW_DEVLOG_FACILITY_VI]         = "VI",
6370         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
6371         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
6372         [FW_DEVLOG_FACILITY_TM]         = "TM",
6373         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
6374         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
6375         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
6376         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
6377         [FW_DEVLOG_FACILITY_RI]         = "RI",
6378         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
6379         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
6380         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
6381         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
6382 };
6383
6384 static int
6385 sysctl_devlog(SYSCTL_HANDLER_ARGS)
6386 {
6387         struct adapter *sc = arg1;
6388         struct devlog_params *dparams = &sc->params.devlog;
6389         struct fw_devlog_e *buf, *e;
6390         int i, j, rc, nentries, first = 0, m;
6391         struct sbuf *sb;
6392         uint64_t ftstamp = UINT64_MAX;
6393
6394         if (dparams->start == 0) {
6395                 dparams->memtype = FW_MEMTYPE_EDC0;
6396                 dparams->start = 0x84000;
6397                 dparams->size = 32768;
6398         }
6399
6400         nentries = dparams->size / sizeof(struct fw_devlog_e);
6401
6402         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
6403         if (buf == NULL)
6404                 return (ENOMEM);
6405
6406         m = fwmtype_to_hwmtype(dparams->memtype);
6407         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
6408         if (rc != 0)
6409                 goto done;
6410
6411         for (i = 0; i < nentries; i++) {
6412                 e = &buf[i];
6413
6414                 if (e->timestamp == 0)
6415                         break;  /* end */
6416
6417                 e->timestamp = be64toh(e->timestamp);
6418                 e->seqno = be32toh(e->seqno);
6419                 for (j = 0; j < 8; j++)
6420                         e->params[j] = be32toh(e->params[j]);
6421
6422                 if (e->timestamp < ftstamp) {
6423                         ftstamp = e->timestamp;
6424                         first = i;
6425                 }
6426         }
6427
6428         if (buf[first].timestamp == 0)
6429                 goto done;      /* nothing in the log */
6430
6431         rc = sysctl_wire_old_buffer(req, 0);
6432         if (rc != 0)
6433                 goto done;
6434
6435         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6436         if (sb == NULL) {
6437                 rc = ENOMEM;
6438                 goto done;
6439         }
6440         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6441             "Seq#", "Tstamp", "Level", "Facility", "Message");
6442
6443         i = first;
6444         do {
6445                 e = &buf[i];
6446                 if (e->timestamp == 0)
6447                         break;  /* end */
6448
6449                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6450                     e->seqno, e->timestamp,
6451                     (e->level < nitems(devlog_level_strings) ?
6452                         devlog_level_strings[e->level] : "UNKNOWN"),
6453                     (e->facility < nitems(devlog_facility_strings) ?
6454                         devlog_facility_strings[e->facility] : "UNKNOWN"));
6455                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6456                     e->params[2], e->params[3], e->params[4],
6457                     e->params[5], e->params[6], e->params[7]);
6458
6459                 if (++i == nentries)
6460                         i = 0;
6461         } while (i != first);
6462
6463         rc = sbuf_finish(sb);
6464         sbuf_delete(sb);
6465 done:
6466         free(buf, M_CXGBE);
6467         return (rc);
6468 }
6469
6470 static int
6471 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6472 {
6473         struct adapter *sc = arg1;
6474         struct sbuf *sb;
6475         int rc;
6476         struct tp_fcoe_stats stats[4];
6477
6478         rc = sysctl_wire_old_buffer(req, 0);
6479         if (rc != 0)
6480                 return (rc);
6481
6482         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6483         if (sb == NULL)
6484                 return (ENOMEM);
6485
6486         t4_get_fcoe_stats(sc, 0, &stats[0]);
6487         t4_get_fcoe_stats(sc, 1, &stats[1]);
6488         t4_get_fcoe_stats(sc, 2, &stats[2]);
6489         t4_get_fcoe_stats(sc, 3, &stats[3]);
6490
6491         sbuf_printf(sb, "                   channel 0        channel 1        "
6492             "channel 2        channel 3\n");
6493         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
6494             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
6495             stats[3].octetsDDP);
6496         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
6497             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
6498         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
6499             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
6500             stats[3].framesDrop);
6501
6502         rc = sbuf_finish(sb);
6503         sbuf_delete(sb);
6504
6505         return (rc);
6506 }
6507
6508 static int
6509 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6510 {
6511         struct adapter *sc = arg1;
6512         struct sbuf *sb;
6513         int rc, i;
6514         unsigned int map, kbps, ipg, mode;
6515         unsigned int pace_tab[NTX_SCHED];
6516
6517         rc = sysctl_wire_old_buffer(req, 0);
6518         if (rc != 0)
6519                 return (rc);
6520
6521         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6522         if (sb == NULL)
6523                 return (ENOMEM);
6524
6525         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6526         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6527         t4_read_pace_tbl(sc, pace_tab);
6528
6529         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6530             "Class IPG (0.1 ns)   Flow IPG (us)");
6531
6532         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6533                 t4_get_tx_sched(sc, i, &kbps, &ipg);
6534                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6535                     (mode & (1 << i)) ? "flow" : "class", map & 3);
6536                 if (kbps)
6537                         sbuf_printf(sb, "%9u     ", kbps);
6538                 else
6539                         sbuf_printf(sb, " disabled     ");
6540
6541                 if (ipg)
6542                         sbuf_printf(sb, "%13u        ", ipg);
6543                 else
6544                         sbuf_printf(sb, "     disabled        ");
6545
6546                 if (pace_tab[i])
6547                         sbuf_printf(sb, "%10u", pace_tab[i]);
6548                 else
6549                         sbuf_printf(sb, "  disabled");
6550         }
6551
6552         rc = sbuf_finish(sb);
6553         sbuf_delete(sb);
6554
6555         return (rc);
6556 }
6557
6558 static int
6559 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6560 {
6561         struct adapter *sc = arg1;
6562         struct sbuf *sb;
6563         int rc, i, j;
6564         uint64_t *p0, *p1;
6565         struct lb_port_stats s[2];
6566         static const char *stat_name[] = {
6567                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6568                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6569                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
6570                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6571                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6572                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6573                 "BG2FramesTrunc:", "BG3FramesTrunc:"
6574         };
6575
6576         rc = sysctl_wire_old_buffer(req, 0);
6577         if (rc != 0)
6578                 return (rc);
6579
6580         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6581         if (sb == NULL)
6582                 return (ENOMEM);
6583
6584         memset(s, 0, sizeof(s));
6585
6586         for (i = 0; i < 4; i += 2) {
6587                 t4_get_lb_stats(sc, i, &s[0]);
6588                 t4_get_lb_stats(sc, i + 1, &s[1]);
6589
6590                 p0 = &s[0].octets;
6591                 p1 = &s[1].octets;
6592                 sbuf_printf(sb, "%s                       Loopback %u"
6593                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6594
6595                 for (j = 0; j < nitems(stat_name); j++)
6596                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6597                                    *p0++, *p1++);
6598         }
6599
6600         rc = sbuf_finish(sb);
6601         sbuf_delete(sb);
6602
6603         return (rc);
6604 }
6605
6606 static int
6607 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6608 {
6609         int rc = 0;
6610         struct port_info *pi = arg1;
6611         struct sbuf *sb;
6612         static const char *linkdnreasons[] = {
6613                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6614                 "PHY overheated", "unknown", "rx los", "reserved7"
6615         };
6616
6617         rc = sysctl_wire_old_buffer(req, 0);
6618         if (rc != 0)
6619                 return(rc);
6620         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6621         if (sb == NULL)
6622                 return (ENOMEM);
6623
6624         if (pi->linkdnrc < 0)
6625                 sbuf_printf(sb, "n/a");
6626         else if (pi->linkdnrc < nitems(linkdnreasons))
6627                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6628         else
6629                 sbuf_printf(sb, "%d", pi->linkdnrc);
6630
6631         rc = sbuf_finish(sb);
6632         sbuf_delete(sb);
6633
6634         return (rc);
6635 }
6636
6637 struct mem_desc {
6638         unsigned int base;
6639         unsigned int limit;
6640         unsigned int idx;
6641 };
6642
6643 static int
6644 mem_desc_cmp(const void *a, const void *b)
6645 {
6646         return ((const struct mem_desc *)a)->base -
6647                ((const struct mem_desc *)b)->base;
6648 }
6649
6650 static void
6651 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6652     unsigned int to)
6653 {
6654         unsigned int size;
6655
6656         size = to - from + 1;
6657         if (size == 0)
6658                 return;
6659
6660         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6661         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6662 }
6663
6664 static int
6665 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6666 {
6667         struct adapter *sc = arg1;
6668         struct sbuf *sb;
6669         int rc, i, n;
6670         uint32_t lo, hi, used, alloc;
6671         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6672         static const char *region[] = {
6673                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6674                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6675                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6676                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6677                 "RQUDP region:", "PBL region:", "TXPBL region:",
6678                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6679                 "On-chip queues:"
6680         };
6681         struct mem_desc avail[4];
6682         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6683         struct mem_desc *md = mem;
6684
6685         rc = sysctl_wire_old_buffer(req, 0);
6686         if (rc != 0)
6687                 return (rc);
6688
6689         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6690         if (sb == NULL)
6691                 return (ENOMEM);
6692
6693         for (i = 0; i < nitems(mem); i++) {
6694                 mem[i].limit = 0;
6695                 mem[i].idx = i;
6696         }
6697
6698         /* Find and sort the populated memory ranges */
6699         i = 0;
6700         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6701         if (lo & F_EDRAM0_ENABLE) {
6702                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6703                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6704                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6705                 avail[i].idx = 0;
6706                 i++;
6707         }
6708         if (lo & F_EDRAM1_ENABLE) {
6709                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6710                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6711                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6712                 avail[i].idx = 1;
6713                 i++;
6714         }
6715         if (lo & F_EXT_MEM_ENABLE) {
6716                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6717                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6718                 avail[i].limit = avail[i].base +
6719                     (G_EXT_MEM_SIZE(hi) << 20);
6720                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6721                 i++;
6722         }
6723         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6724                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6725                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6726                 avail[i].limit = avail[i].base +
6727                     (G_EXT_MEM1_SIZE(hi) << 20);
6728                 avail[i].idx = 4;
6729                 i++;
6730         }
6731         if (!i)                                    /* no memory available */
6732                 return 0;
6733         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6734
6735         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6736         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6737         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6738         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6739         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6740         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6741         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6742         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6743         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6744
6745         /* the next few have explicit upper bounds */
6746         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6747         md->limit = md->base - 1 +
6748                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6749                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6750         md++;
6751
6752         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6753         md->limit = md->base - 1 +
6754                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6755                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6756         md++;
6757
6758         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6759                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6760                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6761                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6762         } else {
6763                 md->base = 0;
6764                 md->idx = nitems(region);  /* hide it */
6765         }
6766         md++;
6767
6768 #define ulp_region(reg) \
6769         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6770         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6771
6772         ulp_region(RX_ISCSI);
6773         ulp_region(RX_TDDP);
6774         ulp_region(TX_TPT);
6775         ulp_region(RX_STAG);
6776         ulp_region(RX_RQ);
6777         ulp_region(RX_RQUDP);
6778         ulp_region(RX_PBL);
6779         ulp_region(TX_PBL);
6780 #undef ulp_region
6781
6782         md->base = 0;
6783         md->idx = nitems(region);
6784         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6785                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6786                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6787                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6788         }
6789         md++;
6790
6791         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6792         md->limit = md->base + sc->tids.ntids - 1;
6793         md++;
6794         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6795         md->limit = md->base + sc->tids.ntids - 1;
6796         md++;
6797
6798         md->base = sc->vres.ocq.start;
6799         if (sc->vres.ocq.size)
6800                 md->limit = md->base + sc->vres.ocq.size - 1;
6801         else
6802                 md->idx = nitems(region);  /* hide it */
6803         md++;
6804
6805         /* add any address-space holes, there can be up to 3 */
6806         for (n = 0; n < i - 1; n++)
6807                 if (avail[n].limit < avail[n + 1].base)
6808                         (md++)->base = avail[n].limit;
6809         if (avail[n].limit)
6810                 (md++)->base = avail[n].limit;
6811
6812         n = md - mem;
6813         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6814
6815         for (lo = 0; lo < i; lo++)
6816                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6817                                 avail[lo].limit - 1);
6818
6819         sbuf_printf(sb, "\n");
6820         for (i = 0; i < n; i++) {
6821                 if (mem[i].idx >= nitems(region))
6822                         continue;                        /* skip holes */
6823                 if (!mem[i].limit)
6824                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6825                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6826                                 mem[i].limit);
6827         }
6828
6829         sbuf_printf(sb, "\n");
6830         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6831         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6832         mem_region_show(sb, "uP RAM:", lo, hi);
6833
6834         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6835         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6836         mem_region_show(sb, "uP Extmem2:", lo, hi);
6837
6838         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6839         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6840                    G_PMRXMAXPAGE(lo),
6841                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6842                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6843
6844         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6845         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6846         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6847                    G_PMTXMAXPAGE(lo),
6848                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6849                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6850         sbuf_printf(sb, "%u p-structs\n",
6851                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6852
6853         for (i = 0; i < 4; i++) {
6854                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6855                 if (is_t4(sc)) {
6856                         used = G_USED(lo);
6857                         alloc = G_ALLOC(lo);
6858                 } else {
6859                         used = G_T5_USED(lo);
6860                         alloc = G_T5_ALLOC(lo);
6861                 }
6862                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6863                            i, used, alloc);
6864         }
6865         for (i = 0; i < 4; i++) {
6866                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6867                 if (is_t4(sc)) {
6868                         used = G_USED(lo);
6869                         alloc = G_ALLOC(lo);
6870                 } else {
6871                         used = G_T5_USED(lo);
6872                         alloc = G_T5_ALLOC(lo);
6873                 }
6874                 sbuf_printf(sb,
6875                            "\nLoopback %d using %u pages out of %u allocated",
6876                            i, used, alloc);
6877         }
6878
6879         rc = sbuf_finish(sb);
6880         sbuf_delete(sb);
6881
6882         return (rc);
6883 }
6884
6885 static inline void
6886 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6887 {
6888         *mask = x | y;
6889         y = htobe64(y);
6890         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6891 }
6892
6893 static int
6894 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6895 {
6896         struct adapter *sc = arg1;
6897         struct sbuf *sb;
6898         int rc, i, n;
6899
6900         rc = sysctl_wire_old_buffer(req, 0);
6901         if (rc != 0)
6902                 return (rc);
6903
6904         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6905         if (sb == NULL)
6906                 return (ENOMEM);
6907
6908         sbuf_printf(sb,
6909             "Idx  Ethernet address     Mask     Vld Ports PF"
6910             "  VF              Replication             P0 P1 P2 P3  ML");
6911         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6912             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6913         for (i = 0; i < n; i++) {
6914                 uint64_t tcamx, tcamy, mask;
6915                 uint32_t cls_lo, cls_hi;
6916                 uint8_t addr[ETHER_ADDR_LEN];
6917
6918                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6919                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6920                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6921                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6922
6923                 if (tcamx & tcamy)
6924                         continue;
6925
6926                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6927                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6928                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6929                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6930                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6931                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6932                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6933
6934                 if (cls_lo & F_REPLICATE) {
6935                         struct fw_ldst_cmd ldst_cmd;
6936
6937                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6938                         ldst_cmd.op_to_addrspace =
6939                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6940                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6941                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6942                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6943                         ldst_cmd.u.mps.rplc.fid_idx =
6944                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6945                                 V_FW_LDST_CMD_IDX(i));
6946
6947                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6948                             "t4mps");
6949                         if (rc)
6950                                 break;
6951                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6952                             sizeof(ldst_cmd), &ldst_cmd);
6953                         end_synchronized_op(sc, 0);
6954
6955                         if (rc != 0) {
6956                                 sbuf_printf(sb,
6957                                     " ------------ error %3u ------------", rc);
6958                                 rc = 0;
6959                         } else {
6960                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6961                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6962                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6963                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6964                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6965                         }
6966                 } else
6967                         sbuf_printf(sb, "%36s", "");
6968
6969                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6970                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6971                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6972         }
6973
6974         if (rc)
6975                 (void) sbuf_finish(sb);
6976         else
6977                 rc = sbuf_finish(sb);
6978         sbuf_delete(sb);
6979
6980         return (rc);
6981 }
6982
6983 static int
6984 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6985 {
6986         struct adapter *sc = arg1;
6987         struct sbuf *sb;
6988         int rc;
6989         uint16_t mtus[NMTUS];
6990
6991         rc = sysctl_wire_old_buffer(req, 0);
6992         if (rc != 0)
6993                 return (rc);
6994
6995         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6996         if (sb == NULL)
6997                 return (ENOMEM);
6998
6999         t4_read_mtu_tbl(sc, mtus, NULL);
7000
7001         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
7002             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
7003             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
7004             mtus[14], mtus[15]);
7005
7006         rc = sbuf_finish(sb);
7007         sbuf_delete(sb);
7008
7009         return (rc);
7010 }
7011
7012 static int
7013 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
7014 {
7015         struct adapter *sc = arg1;
7016         struct sbuf *sb;
7017         int rc, i;
7018         uint32_t cnt[PM_NSTATS];
7019         uint64_t cyc[PM_NSTATS];
7020         static const char *rx_stats[] = {
7021                 "Read:", "Write bypass:", "Write mem:", "Flush:"
7022         };
7023         static const char *tx_stats[] = {
7024                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
7025         };
7026
7027         rc = sysctl_wire_old_buffer(req, 0);
7028         if (rc != 0)
7029                 return (rc);
7030
7031         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7032         if (sb == NULL)
7033                 return (ENOMEM);
7034
7035         t4_pmtx_get_stats(sc, cnt, cyc);
7036         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
7037         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
7038                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
7039                     cyc[i]);
7040
7041         t4_pmrx_get_stats(sc, cnt, cyc);
7042         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
7043         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
7044                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
7045                     cyc[i]);
7046
7047         rc = sbuf_finish(sb);
7048         sbuf_delete(sb);
7049
7050         return (rc);
7051 }
7052
7053 static int
7054 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
7055 {
7056         struct adapter *sc = arg1;
7057         struct sbuf *sb;
7058         int rc;
7059         struct tp_rdma_stats stats;
7060
7061         rc = sysctl_wire_old_buffer(req, 0);
7062         if (rc != 0)
7063                 return (rc);
7064
7065         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7066         if (sb == NULL)
7067                 return (ENOMEM);
7068
7069         t4_tp_get_rdma_stats(sc, &stats);
7070         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
7071         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
7072
7073         rc = sbuf_finish(sb);
7074         sbuf_delete(sb);
7075
7076         return (rc);
7077 }
7078
7079 static int
7080 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
7081 {
7082         struct adapter *sc = arg1;
7083         struct sbuf *sb;
7084         int rc;
7085         struct tp_tcp_stats v4, v6;
7086
7087         rc = sysctl_wire_old_buffer(req, 0);
7088         if (rc != 0)
7089                 return (rc);
7090
7091         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7092         if (sb == NULL)
7093                 return (ENOMEM);
7094
7095         t4_tp_get_tcp_stats(sc, &v4, &v6);
7096         sbuf_printf(sb,
7097             "                                IP                 IPv6\n");
7098         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
7099             v4.tcpOutRsts, v6.tcpOutRsts);
7100         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
7101             v4.tcpInSegs, v6.tcpInSegs);
7102         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
7103             v4.tcpOutSegs, v6.tcpOutSegs);
7104         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
7105             v4.tcpRetransSegs, v6.tcpRetransSegs);
7106
7107         rc = sbuf_finish(sb);
7108         sbuf_delete(sb);
7109
7110         return (rc);
7111 }
7112
7113 static int
7114 sysctl_tids(SYSCTL_HANDLER_ARGS)
7115 {
7116         struct adapter *sc = arg1;
7117         struct sbuf *sb;
7118         int rc;
7119         struct tid_info *t = &sc->tids;
7120
7121         rc = sysctl_wire_old_buffer(req, 0);
7122         if (rc != 0)
7123                 return (rc);
7124
7125         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7126         if (sb == NULL)
7127                 return (ENOMEM);
7128
7129         if (t->natids) {
7130                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
7131                     t->atids_in_use);
7132         }
7133
7134         if (t->ntids) {
7135                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7136                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
7137
7138                         if (b) {
7139                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
7140                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7141                                     t->ntids - 1);
7142                         } else {
7143                                 sbuf_printf(sb, "TID range: %u-%u",
7144                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7145                                     t->ntids - 1);
7146                         }
7147                 } else
7148                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
7149                 sbuf_printf(sb, ", in use: %u\n",
7150                     atomic_load_acq_int(&t->tids_in_use));
7151         }
7152
7153         if (t->nstids) {
7154                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
7155                     t->stid_base + t->nstids - 1, t->stids_in_use);
7156         }
7157
7158         if (t->nftids) {
7159                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
7160                     t->ftid_base + t->nftids - 1);
7161         }
7162
7163         if (t->netids) {
7164                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
7165                     t->etid_base + t->netids - 1);
7166         }
7167
7168         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
7169             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
7170             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
7171
7172         rc = sbuf_finish(sb);
7173         sbuf_delete(sb);
7174
7175         return (rc);
7176 }
7177
7178 static int
7179 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
7180 {
7181         struct adapter *sc = arg1;
7182         struct sbuf *sb;
7183         int rc;
7184         struct tp_err_stats stats;
7185
7186         rc = sysctl_wire_old_buffer(req, 0);
7187         if (rc != 0)
7188                 return (rc);
7189
7190         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7191         if (sb == NULL)
7192                 return (ENOMEM);
7193
7194         t4_tp_get_err_stats(sc, &stats);
7195
7196         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
7197                       "channel 3\n");
7198         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
7199             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
7200             stats.macInErrs[3]);
7201         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
7202             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
7203             stats.hdrInErrs[3]);
7204         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
7205             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
7206             stats.tcpInErrs[3]);
7207         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
7208             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
7209             stats.tcp6InErrs[3]);
7210         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
7211             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
7212             stats.tnlCongDrops[3]);
7213         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
7214             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
7215             stats.tnlTxDrops[3]);
7216         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
7217             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
7218             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
7219         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
7220             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
7221             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
7222         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
7223             stats.ofldNoNeigh, stats.ofldCongDefer);
7224
7225         rc = sbuf_finish(sb);
7226         sbuf_delete(sb);
7227
7228         return (rc);
7229 }
7230
7231 struct field_desc {
7232         const char *name;
7233         u_int start;
7234         u_int width;
7235 };
7236
7237 static void
7238 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
7239 {
7240         char buf[32];
7241         int line_size = 0;
7242
7243         while (f->name) {
7244                 uint64_t mask = (1ULL << f->width) - 1;
7245                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
7246                     ((uintmax_t)v >> f->start) & mask);
7247
7248                 if (line_size + len >= 79) {
7249                         line_size = 8;
7250                         sbuf_printf(sb, "\n        ");
7251                 }
7252                 sbuf_printf(sb, "%s ", buf);
7253                 line_size += len + 1;
7254                 f++;
7255         }
7256         sbuf_printf(sb, "\n");
7257 }
7258
7259 static struct field_desc tp_la0[] = {
7260         { "RcfOpCodeOut", 60, 4 },
7261         { "State", 56, 4 },
7262         { "WcfState", 52, 4 },
7263         { "RcfOpcSrcOut", 50, 2 },
7264         { "CRxError", 49, 1 },
7265         { "ERxError", 48, 1 },
7266         { "SanityFailed", 47, 1 },
7267         { "SpuriousMsg", 46, 1 },
7268         { "FlushInputMsg", 45, 1 },
7269         { "FlushInputCpl", 44, 1 },
7270         { "RssUpBit", 43, 1 },
7271         { "RssFilterHit", 42, 1 },
7272         { "Tid", 32, 10 },
7273         { "InitTcb", 31, 1 },
7274         { "LineNumber", 24, 7 },
7275         { "Emsg", 23, 1 },
7276         { "EdataOut", 22, 1 },
7277         { "Cmsg", 21, 1 },
7278         { "CdataOut", 20, 1 },
7279         { "EreadPdu", 19, 1 },
7280         { "CreadPdu", 18, 1 },
7281         { "TunnelPkt", 17, 1 },
7282         { "RcfPeerFin", 16, 1 },
7283         { "RcfReasonOut", 12, 4 },
7284         { "TxCchannel", 10, 2 },
7285         { "RcfTxChannel", 8, 2 },
7286         { "RxEchannel", 6, 2 },
7287         { "RcfRxChannel", 5, 1 },
7288         { "RcfDataOutSrdy", 4, 1 },
7289         { "RxDvld", 3, 1 },
7290         { "RxOoDvld", 2, 1 },
7291         { "RxCongestion", 1, 1 },
7292         { "TxCongestion", 0, 1 },
7293         { NULL }
7294 };
7295
7296 static struct field_desc tp_la1[] = {
7297         { "CplCmdIn", 56, 8 },
7298         { "CplCmdOut", 48, 8 },
7299         { "ESynOut", 47, 1 },
7300         { "EAckOut", 46, 1 },
7301         { "EFinOut", 45, 1 },
7302         { "ERstOut", 44, 1 },
7303         { "SynIn", 43, 1 },
7304         { "AckIn", 42, 1 },
7305         { "FinIn", 41, 1 },
7306         { "RstIn", 40, 1 },
7307         { "DataIn", 39, 1 },
7308         { "DataInVld", 38, 1 },
7309         { "PadIn", 37, 1 },
7310         { "RxBufEmpty", 36, 1 },
7311         { "RxDdp", 35, 1 },
7312         { "RxFbCongestion", 34, 1 },
7313         { "TxFbCongestion", 33, 1 },
7314         { "TxPktSumSrdy", 32, 1 },
7315         { "RcfUlpType", 28, 4 },
7316         { "Eread", 27, 1 },
7317         { "Ebypass", 26, 1 },
7318         { "Esave", 25, 1 },
7319         { "Static0", 24, 1 },
7320         { "Cread", 23, 1 },
7321         { "Cbypass", 22, 1 },
7322         { "Csave", 21, 1 },
7323         { "CPktOut", 20, 1 },
7324         { "RxPagePoolFull", 18, 2 },
7325         { "RxLpbkPkt", 17, 1 },
7326         { "TxLpbkPkt", 16, 1 },
7327         { "RxVfValid", 15, 1 },
7328         { "SynLearned", 14, 1 },
7329         { "SetDelEntry", 13, 1 },
7330         { "SetInvEntry", 12, 1 },
7331         { "CpcmdDvld", 11, 1 },
7332         { "CpcmdSave", 10, 1 },
7333         { "RxPstructsFull", 8, 2 },
7334         { "EpcmdDvld", 7, 1 },
7335         { "EpcmdFlush", 6, 1 },
7336         { "EpcmdTrimPrefix", 5, 1 },
7337         { "EpcmdTrimPostfix", 4, 1 },
7338         { "ERssIp4Pkt", 3, 1 },
7339         { "ERssIp6Pkt", 2, 1 },
7340         { "ERssTcpUdpPkt", 1, 1 },
7341         { "ERssFceFipPkt", 0, 1 },
7342         { NULL }
7343 };
7344
7345 static struct field_desc tp_la2[] = {
7346         { "CplCmdIn", 56, 8 },
7347         { "MpsVfVld", 55, 1 },
7348         { "MpsPf", 52, 3 },
7349         { "MpsVf", 44, 8 },
7350         { "SynIn", 43, 1 },
7351         { "AckIn", 42, 1 },
7352         { "FinIn", 41, 1 },
7353         { "RstIn", 40, 1 },
7354         { "DataIn", 39, 1 },
7355         { "DataInVld", 38, 1 },
7356         { "PadIn", 37, 1 },
7357         { "RxBufEmpty", 36, 1 },
7358         { "RxDdp", 35, 1 },
7359         { "RxFbCongestion", 34, 1 },
7360         { "TxFbCongestion", 33, 1 },
7361         { "TxPktSumSrdy", 32, 1 },
7362         { "RcfUlpType", 28, 4 },
7363         { "Eread", 27, 1 },
7364         { "Ebypass", 26, 1 },
7365         { "Esave", 25, 1 },
7366         { "Static0", 24, 1 },
7367         { "Cread", 23, 1 },
7368         { "Cbypass", 22, 1 },
7369         { "Csave", 21, 1 },
7370         { "CPktOut", 20, 1 },
7371         { "RxPagePoolFull", 18, 2 },
7372         { "RxLpbkPkt", 17, 1 },
7373         { "TxLpbkPkt", 16, 1 },
7374         { "RxVfValid", 15, 1 },
7375         { "SynLearned", 14, 1 },
7376         { "SetDelEntry", 13, 1 },
7377         { "SetInvEntry", 12, 1 },
7378         { "CpcmdDvld", 11, 1 },
7379         { "CpcmdSave", 10, 1 },
7380         { "RxPstructsFull", 8, 2 },
7381         { "EpcmdDvld", 7, 1 },
7382         { "EpcmdFlush", 6, 1 },
7383         { "EpcmdTrimPrefix", 5, 1 },
7384         { "EpcmdTrimPostfix", 4, 1 },
7385         { "ERssIp4Pkt", 3, 1 },
7386         { "ERssIp6Pkt", 2, 1 },
7387         { "ERssTcpUdpPkt", 1, 1 },
7388         { "ERssFceFipPkt", 0, 1 },
7389         { NULL }
7390 };
7391
7392 static void
7393 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
7394 {
7395
7396         field_desc_show(sb, *p, tp_la0);
7397 }
7398
7399 static void
7400 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
7401 {
7402
7403         if (idx)
7404                 sbuf_printf(sb, "\n");
7405         field_desc_show(sb, p[0], tp_la0);
7406         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7407                 field_desc_show(sb, p[1], tp_la0);
7408 }
7409
7410 static void
7411 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
7412 {
7413
7414         if (idx)
7415                 sbuf_printf(sb, "\n");
7416         field_desc_show(sb, p[0], tp_la0);
7417         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7418                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7419 }
7420
7421 static int
7422 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7423 {
7424         struct adapter *sc = arg1;
7425         struct sbuf *sb;
7426         uint64_t *buf, *p;
7427         int rc;
7428         u_int i, inc;
7429         void (*show_func)(struct sbuf *, uint64_t *, int);
7430
7431         rc = sysctl_wire_old_buffer(req, 0);
7432         if (rc != 0)
7433                 return (rc);
7434
7435         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7436         if (sb == NULL)
7437                 return (ENOMEM);
7438
7439         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7440
7441         t4_tp_read_la(sc, buf, NULL);
7442         p = buf;
7443
7444         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7445         case 2:
7446                 inc = 2;
7447                 show_func = tp_la_show2;
7448                 break;
7449         case 3:
7450                 inc = 2;
7451                 show_func = tp_la_show3;
7452                 break;
7453         default:
7454                 inc = 1;
7455                 show_func = tp_la_show;
7456         }
7457
7458         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7459                 (*show_func)(sb, p, i);
7460
7461         rc = sbuf_finish(sb);
7462         sbuf_delete(sb);
7463         free(buf, M_CXGBE);
7464         return (rc);
7465 }
7466
7467 static int
7468 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7469 {
7470         struct adapter *sc = arg1;
7471         struct sbuf *sb;
7472         int rc;
7473         u64 nrate[NCHAN], orate[NCHAN];
7474
7475         rc = sysctl_wire_old_buffer(req, 0);
7476         if (rc != 0)
7477                 return (rc);
7478
7479         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7480         if (sb == NULL)
7481                 return (ENOMEM);
7482
7483         t4_get_chan_txrate(sc, nrate, orate);
7484         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
7485                  "channel 3\n");
7486         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7487             nrate[0], nrate[1], nrate[2], nrate[3]);
7488         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7489             orate[0], orate[1], orate[2], orate[3]);
7490
7491         rc = sbuf_finish(sb);
7492         sbuf_delete(sb);
7493
7494         return (rc);
7495 }
7496
7497 static int
7498 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7499 {
7500         struct adapter *sc = arg1;
7501         struct sbuf *sb;
7502         uint32_t *buf, *p;
7503         int rc, i;
7504
7505         rc = sysctl_wire_old_buffer(req, 0);
7506         if (rc != 0)
7507                 return (rc);
7508
7509         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7510         if (sb == NULL)
7511                 return (ENOMEM);
7512
7513         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7514             M_ZERO | M_WAITOK);
7515
7516         t4_ulprx_read_la(sc, buf);
7517         p = buf;
7518
7519         sbuf_printf(sb, "      Pcmd        Type   Message"
7520             "                Data");
7521         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7522                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7523                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7524         }
7525
7526         rc = sbuf_finish(sb);
7527         sbuf_delete(sb);
7528         free(buf, M_CXGBE);
7529         return (rc);
7530 }
7531
7532 static int
7533 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7534 {
7535         struct adapter *sc = arg1;
7536         struct sbuf *sb;
7537         int rc, v;
7538
7539         rc = sysctl_wire_old_buffer(req, 0);
7540         if (rc != 0)
7541                 return (rc);
7542
7543         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7544         if (sb == NULL)
7545                 return (ENOMEM);
7546
7547         v = t4_read_reg(sc, A_SGE_STAT_CFG);
7548         if (G_STATSOURCE_T5(v) == 7) {
7549                 if (G_STATMODE(v) == 0) {
7550                         sbuf_printf(sb, "total %d, incomplete %d",
7551                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7552                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7553                 } else if (G_STATMODE(v) == 1) {
7554                         sbuf_printf(sb, "total %d, data overflow %d",
7555                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7556                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7557                 }
7558         }
7559         rc = sbuf_finish(sb);
7560         sbuf_delete(sb);
7561
7562         return (rc);
7563 }
7564 #endif
7565
7566 static uint32_t
7567 fconf_to_mode(uint32_t fconf)
7568 {
7569         uint32_t mode;
7570
7571         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7572             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7573
7574         if (fconf & F_FRAGMENTATION)
7575                 mode |= T4_FILTER_IP_FRAGMENT;
7576
7577         if (fconf & F_MPSHITTYPE)
7578                 mode |= T4_FILTER_MPS_HIT_TYPE;
7579
7580         if (fconf & F_MACMATCH)
7581                 mode |= T4_FILTER_MAC_IDX;
7582
7583         if (fconf & F_ETHERTYPE)
7584                 mode |= T4_FILTER_ETH_TYPE;
7585
7586         if (fconf & F_PROTOCOL)
7587                 mode |= T4_FILTER_IP_PROTO;
7588
7589         if (fconf & F_TOS)
7590                 mode |= T4_FILTER_IP_TOS;
7591
7592         if (fconf & F_VLAN)
7593                 mode |= T4_FILTER_VLAN;
7594
7595         if (fconf & F_VNIC_ID)
7596                 mode |= T4_FILTER_VNIC;
7597
7598         if (fconf & F_PORT)
7599                 mode |= T4_FILTER_PORT;
7600
7601         if (fconf & F_FCOE)
7602                 mode |= T4_FILTER_FCoE;
7603
7604         return (mode);
7605 }
7606
7607 static uint32_t
7608 mode_to_fconf(uint32_t mode)
7609 {
7610         uint32_t fconf = 0;
7611
7612         if (mode & T4_FILTER_IP_FRAGMENT)
7613                 fconf |= F_FRAGMENTATION;
7614
7615         if (mode & T4_FILTER_MPS_HIT_TYPE)
7616                 fconf |= F_MPSHITTYPE;
7617
7618         if (mode & T4_FILTER_MAC_IDX)
7619                 fconf |= F_MACMATCH;
7620
7621         if (mode & T4_FILTER_ETH_TYPE)
7622                 fconf |= F_ETHERTYPE;
7623
7624         if (mode & T4_FILTER_IP_PROTO)
7625                 fconf |= F_PROTOCOL;
7626
7627         if (mode & T4_FILTER_IP_TOS)
7628                 fconf |= F_TOS;
7629
7630         if (mode & T4_FILTER_VLAN)
7631                 fconf |= F_VLAN;
7632
7633         if (mode & T4_FILTER_VNIC)
7634                 fconf |= F_VNIC_ID;
7635
7636         if (mode & T4_FILTER_PORT)
7637                 fconf |= F_PORT;
7638
7639         if (mode & T4_FILTER_FCoE)
7640                 fconf |= F_FCOE;
7641
7642         return (fconf);
7643 }
7644
7645 static uint32_t
7646 fspec_to_fconf(struct t4_filter_specification *fs)
7647 {
7648         uint32_t fconf = 0;
7649
7650         if (fs->val.frag || fs->mask.frag)
7651                 fconf |= F_FRAGMENTATION;
7652
7653         if (fs->val.matchtype || fs->mask.matchtype)
7654                 fconf |= F_MPSHITTYPE;
7655
7656         if (fs->val.macidx || fs->mask.macidx)
7657                 fconf |= F_MACMATCH;
7658
7659         if (fs->val.ethtype || fs->mask.ethtype)
7660                 fconf |= F_ETHERTYPE;
7661
7662         if (fs->val.proto || fs->mask.proto)
7663                 fconf |= F_PROTOCOL;
7664
7665         if (fs->val.tos || fs->mask.tos)
7666                 fconf |= F_TOS;
7667
7668         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7669                 fconf |= F_VLAN;
7670
7671         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7672                 fconf |= F_VNIC_ID;
7673
7674         if (fs->val.iport || fs->mask.iport)
7675                 fconf |= F_PORT;
7676
7677         if (fs->val.fcoe || fs->mask.fcoe)
7678                 fconf |= F_FCOE;
7679
7680         return (fconf);
7681 }
7682
7683 static int
7684 get_filter_mode(struct adapter *sc, uint32_t *mode)
7685 {
7686         int rc;
7687         uint32_t fconf;
7688
7689         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7690             "t4getfm");
7691         if (rc)
7692                 return (rc);
7693
7694         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7695             A_TP_VLAN_PRI_MAP);
7696
7697         if (sc->params.tp.vlan_pri_map != fconf) {
7698                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7699                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7700                     fconf);
7701         }
7702
7703         *mode = fconf_to_mode(fconf);
7704
7705         end_synchronized_op(sc, LOCK_HELD);
7706         return (0);
7707 }
7708
7709 static int
7710 set_filter_mode(struct adapter *sc, uint32_t mode)
7711 {
7712         uint32_t fconf;
7713         int rc;
7714
7715         fconf = mode_to_fconf(mode);
7716
7717         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7718             "t4setfm");
7719         if (rc)
7720                 return (rc);
7721
7722         if (sc->tids.ftids_in_use > 0) {
7723                 rc = EBUSY;
7724                 goto done;
7725         }
7726
7727 #ifdef TCP_OFFLOAD
7728         if (uld_active(sc, ULD_TOM)) {
7729                 rc = EBUSY;
7730                 goto done;
7731         }
7732 #endif
7733
7734         rc = -t4_set_filter_mode(sc, fconf);
7735 done:
7736         end_synchronized_op(sc, LOCK_HELD);
7737         return (rc);
7738 }
7739
7740 static inline uint64_t
7741 get_filter_hits(struct adapter *sc, uint32_t fid)
7742 {
7743         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7744         uint64_t hits;
7745
7746         memwin_info(sc, 0, &mw_base, NULL);
7747         off = position_memwin(sc, 0,
7748             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7749         if (is_t4(sc)) {
7750                 hits = t4_read_reg64(sc, mw_base + off + 16);
7751                 hits = be64toh(hits);
7752         } else {
7753                 hits = t4_read_reg(sc, mw_base + off + 24);
7754                 hits = be32toh(hits);
7755         }
7756
7757         return (hits);
7758 }
7759
7760 static int
7761 get_filter(struct adapter *sc, struct t4_filter *t)
7762 {
7763         int i, rc, nfilters = sc->tids.nftids;
7764         struct filter_entry *f;
7765
7766         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7767             "t4getf");
7768         if (rc)
7769                 return (rc);
7770
7771         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7772             t->idx >= nfilters) {
7773                 t->idx = 0xffffffff;
7774                 goto done;
7775         }
7776
7777         f = &sc->tids.ftid_tab[t->idx];
7778         for (i = t->idx; i < nfilters; i++, f++) {
7779                 if (f->valid) {
7780                         t->idx = i;
7781                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7782                         t->smtidx = f->smtidx;
7783                         if (f->fs.hitcnts)
7784                                 t->hits = get_filter_hits(sc, t->idx);
7785                         else
7786                                 t->hits = UINT64_MAX;
7787                         t->fs = f->fs;
7788
7789                         goto done;
7790                 }
7791         }
7792
7793         t->idx = 0xffffffff;
7794 done:
7795         end_synchronized_op(sc, LOCK_HELD);
7796         return (0);
7797 }
7798
7799 static int
7800 set_filter(struct adapter *sc, struct t4_filter *t)
7801 {
7802         unsigned int nfilters, nports;
7803         struct filter_entry *f;
7804         int i, rc;
7805
7806         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7807         if (rc)
7808                 return (rc);
7809
7810         nfilters = sc->tids.nftids;
7811         nports = sc->params.nports;
7812
7813         if (nfilters == 0) {
7814                 rc = ENOTSUP;
7815                 goto done;
7816         }
7817
7818         if (!(sc->flags & FULL_INIT_DONE)) {
7819                 rc = EAGAIN;
7820                 goto done;
7821         }
7822
7823         if (t->idx >= nfilters) {
7824                 rc = EINVAL;
7825                 goto done;
7826         }
7827
7828         /* Validate against the global filter mode */
7829         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7830             sc->params.tp.vlan_pri_map) {
7831                 rc = E2BIG;
7832                 goto done;
7833         }
7834
7835         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7836                 rc = EINVAL;
7837                 goto done;
7838         }
7839
7840         if (t->fs.val.iport >= nports) {
7841                 rc = EINVAL;
7842                 goto done;
7843         }
7844
7845         /* Can't specify an iq if not steering to it */
7846         if (!t->fs.dirsteer && t->fs.iq) {
7847                 rc = EINVAL;
7848                 goto done;
7849         }
7850
7851         /* IPv6 filter idx must be 4 aligned */
7852         if (t->fs.type == 1 &&
7853             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7854                 rc = EINVAL;
7855                 goto done;
7856         }
7857
7858         if (sc->tids.ftid_tab == NULL) {
7859                 KASSERT(sc->tids.ftids_in_use == 0,
7860                     ("%s: no memory allocated but filters_in_use > 0",
7861                     __func__));
7862
7863                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7864                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7865                 if (sc->tids.ftid_tab == NULL) {
7866                         rc = ENOMEM;
7867                         goto done;
7868                 }
7869                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7870         }
7871
7872         for (i = 0; i < 4; i++) {
7873                 f = &sc->tids.ftid_tab[t->idx + i];
7874
7875                 if (f->pending || f->valid) {
7876                         rc = EBUSY;
7877                         goto done;
7878                 }
7879                 if (f->locked) {
7880                         rc = EPERM;
7881                         goto done;
7882                 }
7883
7884                 if (t->fs.type == 0)
7885                         break;
7886         }
7887
7888         f = &sc->tids.ftid_tab[t->idx];
7889         f->fs = t->fs;
7890
7891         rc = set_filter_wr(sc, t->idx);
7892 done:
7893         end_synchronized_op(sc, 0);
7894
7895         if (rc == 0) {
7896                 mtx_lock(&sc->tids.ftid_lock);
7897                 for (;;) {
7898                         if (f->pending == 0) {
7899                                 rc = f->valid ? 0 : EIO;
7900                                 break;
7901                         }
7902
7903                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7904                             PCATCH, "t4setfw", 0)) {
7905                                 rc = EINPROGRESS;
7906                                 break;
7907                         }
7908                 }
7909                 mtx_unlock(&sc->tids.ftid_lock);
7910         }
7911         return (rc);
7912 }
7913
7914 static int
7915 del_filter(struct adapter *sc, struct t4_filter *t)
7916 {
7917         unsigned int nfilters;
7918         struct filter_entry *f;
7919         int rc;
7920
7921         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7922         if (rc)
7923                 return (rc);
7924
7925         nfilters = sc->tids.nftids;
7926
7927         if (nfilters == 0) {
7928                 rc = ENOTSUP;
7929                 goto done;
7930         }
7931
7932         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7933             t->idx >= nfilters) {
7934                 rc = EINVAL;
7935                 goto done;
7936         }
7937
7938         if (!(sc->flags & FULL_INIT_DONE)) {
7939                 rc = EAGAIN;
7940                 goto done;
7941         }
7942
7943         f = &sc->tids.ftid_tab[t->idx];
7944
7945         if (f->pending) {
7946                 rc = EBUSY;
7947                 goto done;
7948         }
7949         if (f->locked) {
7950                 rc = EPERM;
7951                 goto done;
7952         }
7953
7954         if (f->valid) {
7955                 t->fs = f->fs;  /* extra info for the caller */
7956                 rc = del_filter_wr(sc, t->idx);
7957         }
7958
7959 done:
7960         end_synchronized_op(sc, 0);
7961
7962         if (rc == 0) {
7963                 mtx_lock(&sc->tids.ftid_lock);
7964                 for (;;) {
7965                         if (f->pending == 0) {
7966                                 rc = f->valid ? EIO : 0;
7967                                 break;
7968                         }
7969
7970                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7971                             PCATCH, "t4delfw", 0)) {
7972                                 rc = EINPROGRESS;
7973                                 break;
7974                         }
7975                 }
7976                 mtx_unlock(&sc->tids.ftid_lock);
7977         }
7978
7979         return (rc);
7980 }
7981
7982 static void
7983 clear_filter(struct filter_entry *f)
7984 {
7985         if (f->l2t)
7986                 t4_l2t_release(f->l2t);
7987
7988         bzero(f, sizeof (*f));
7989 }
7990
7991 static int
7992 set_filter_wr(struct adapter *sc, int fidx)
7993 {
7994         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7995         struct fw_filter_wr *fwr;
7996         unsigned int ftid;
7997         struct wrq_cookie cookie;
7998
7999         ASSERT_SYNCHRONIZED_OP(sc);
8000
8001         if (f->fs.newdmac || f->fs.newvlan) {
8002                 /* This filter needs an L2T entry; allocate one. */
8003                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
8004                 if (f->l2t == NULL)
8005                         return (EAGAIN);
8006                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
8007                     f->fs.dmac)) {
8008                         t4_l2t_release(f->l2t);
8009                         f->l2t = NULL;
8010                         return (ENOMEM);
8011                 }
8012         }
8013
8014         ftid = sc->tids.ftid_base + fidx;
8015
8016         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8017         if (fwr == NULL)
8018                 return (ENOMEM);
8019         bzero(fwr, sizeof(*fwr));
8020
8021         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
8022         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
8023         fwr->tid_to_iq =
8024             htobe32(V_FW_FILTER_WR_TID(ftid) |
8025                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
8026                 V_FW_FILTER_WR_NOREPLY(0) |
8027                 V_FW_FILTER_WR_IQ(f->fs.iq));
8028         fwr->del_filter_to_l2tix =
8029             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
8030                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
8031                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
8032                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
8033                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
8034                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
8035                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
8036                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
8037                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
8038                     f->fs.newvlan == VLAN_REWRITE) |
8039                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
8040                     f->fs.newvlan == VLAN_REWRITE) |
8041                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
8042                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
8043                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
8044                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
8045         fwr->ethtype = htobe16(f->fs.val.ethtype);
8046         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
8047         fwr->frag_to_ovlan_vldm =
8048             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
8049                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
8050                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
8051                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
8052                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
8053                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
8054         fwr->smac_sel = 0;
8055         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
8056             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
8057         fwr->maci_to_matchtypem =
8058             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
8059                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
8060                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
8061                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
8062                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
8063                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
8064                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
8065                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
8066         fwr->ptcl = f->fs.val.proto;
8067         fwr->ptclm = f->fs.mask.proto;
8068         fwr->ttyp = f->fs.val.tos;
8069         fwr->ttypm = f->fs.mask.tos;
8070         fwr->ivlan = htobe16(f->fs.val.vlan);
8071         fwr->ivlanm = htobe16(f->fs.mask.vlan);
8072         fwr->ovlan = htobe16(f->fs.val.vnic);
8073         fwr->ovlanm = htobe16(f->fs.mask.vnic);
8074         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
8075         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
8076         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
8077         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
8078         fwr->lp = htobe16(f->fs.val.dport);
8079         fwr->lpm = htobe16(f->fs.mask.dport);
8080         fwr->fp = htobe16(f->fs.val.sport);
8081         fwr->fpm = htobe16(f->fs.mask.sport);
8082         if (f->fs.newsmac)
8083                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
8084
8085         f->pending = 1;
8086         sc->tids.ftids_in_use++;
8087
8088         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8089         return (0);
8090 }
8091
8092 static int
8093 del_filter_wr(struct adapter *sc, int fidx)
8094 {
8095         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8096         struct fw_filter_wr *fwr;
8097         unsigned int ftid;
8098         struct wrq_cookie cookie;
8099
8100         ftid = sc->tids.ftid_base + fidx;
8101
8102         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8103         if (fwr == NULL)
8104                 return (ENOMEM);
8105         bzero(fwr, sizeof (*fwr));
8106
8107         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
8108
8109         f->pending = 1;
8110         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8111         return (0);
8112 }
8113
8114 int
8115 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8116 {
8117         struct adapter *sc = iq->adapter;
8118         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
8119         unsigned int idx = GET_TID(rpl);
8120         unsigned int rc;
8121         struct filter_entry *f;
8122
8123         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
8124             rss->opcode));
8125
8126         if (is_ftid(sc, idx)) {
8127
8128                 idx -= sc->tids.ftid_base;
8129                 f = &sc->tids.ftid_tab[idx];
8130                 rc = G_COOKIE(rpl->cookie);
8131
8132                 mtx_lock(&sc->tids.ftid_lock);
8133                 if (rc == FW_FILTER_WR_FLT_ADDED) {
8134                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
8135                             __func__, idx));
8136                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
8137                         f->pending = 0;  /* asynchronous setup completed */
8138                         f->valid = 1;
8139                 } else {
8140                         if (rc != FW_FILTER_WR_FLT_DELETED) {
8141                                 /* Add or delete failed, display an error */
8142                                 log(LOG_ERR,
8143                                     "filter %u setup failed with error %u\n",
8144                                     idx, rc);
8145                         }
8146
8147                         clear_filter(f);
8148                         sc->tids.ftids_in_use--;
8149                 }
8150                 wakeup(&sc->tids.ftid_tab);
8151                 mtx_unlock(&sc->tids.ftid_lock);
8152         }
8153
8154         return (0);
8155 }
8156
8157 static int
8158 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
8159 {
8160         int rc;
8161
8162         if (cntxt->cid > M_CTXTQID)
8163                 return (EINVAL);
8164
8165         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
8166             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
8167                 return (EINVAL);
8168
8169         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
8170         if (rc)
8171                 return (rc);
8172
8173         if (sc->flags & FW_OK) {
8174                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
8175                     &cntxt->data[0]);
8176                 if (rc == 0)
8177                         goto done;
8178         }
8179
8180         /*
8181          * Read via firmware failed or wasn't even attempted.  Read directly via
8182          * the backdoor.
8183          */
8184         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
8185 done:
8186         end_synchronized_op(sc, 0);
8187         return (rc);
8188 }
8189
8190 static int
8191 load_fw(struct adapter *sc, struct t4_data *fw)
8192 {
8193         int rc;
8194         uint8_t *fw_data;
8195
8196         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
8197         if (rc)
8198                 return (rc);
8199
8200         if (sc->flags & FULL_INIT_DONE) {
8201                 rc = EBUSY;
8202                 goto done;
8203         }
8204
8205         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
8206         if (fw_data == NULL) {
8207                 rc = ENOMEM;
8208                 goto done;
8209         }
8210
8211         rc = copyin(fw->data, fw_data, fw->len);
8212         if (rc == 0)
8213                 rc = -t4_load_fw(sc, fw_data, fw->len);
8214
8215         free(fw_data, M_CXGBE);
8216 done:
8217         end_synchronized_op(sc, 0);
8218         return (rc);
8219 }
8220
8221 static int
8222 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
8223 {
8224         uint32_t addr, off, remaining, i, n;
8225         uint32_t *buf, *b;
8226         uint32_t mw_base, mw_aperture;
8227         int rc;
8228         uint8_t *dst;
8229
8230         rc = validate_mem_range(sc, mr->addr, mr->len);
8231         if (rc != 0)
8232                 return (rc);
8233
8234         memwin_info(sc, win, &mw_base, &mw_aperture);
8235         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
8236         addr = mr->addr;
8237         remaining = mr->len;
8238         dst = (void *)mr->data;
8239
8240         while (remaining) {
8241                 off = position_memwin(sc, win, addr);
8242
8243                 /* number of bytes that we'll copy in the inner loop */
8244                 n = min(remaining, mw_aperture - off);
8245                 for (i = 0; i < n; i += 4)
8246                         *b++ = t4_read_reg(sc, mw_base + off + i);
8247
8248                 rc = copyout(buf, dst, n);
8249                 if (rc != 0)
8250                         break;
8251
8252                 b = buf;
8253                 dst += n;
8254                 remaining -= n;
8255                 addr += n;
8256         }
8257
8258         free(buf, M_CXGBE);
8259         return (rc);
8260 }
8261
8262 static int
8263 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
8264 {
8265         int rc;
8266
8267         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
8268                 return (EINVAL);
8269
8270         if (i2cd->len > sizeof(i2cd->data))
8271                 return (EFBIG);
8272
8273         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
8274         if (rc)
8275                 return (rc);
8276         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
8277             i2cd->offset, i2cd->len, &i2cd->data[0]);
8278         end_synchronized_op(sc, 0);
8279
8280         return (rc);
8281 }
8282
8283 static int
8284 in_range(int val, int lo, int hi)
8285 {
8286
8287         return (val < 0 || (val <= hi && val >= lo));
8288 }
8289
8290 static int
8291 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
8292 {
8293         int fw_subcmd, fw_type, rc;
8294
8295         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
8296         if (rc)
8297                 return (rc);
8298
8299         if (!(sc->flags & FULL_INIT_DONE)) {
8300                 rc = EAGAIN;
8301                 goto done;
8302         }
8303
8304         /*
8305          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
8306          * sub-command and type are in common locations.)
8307          */
8308         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
8309                 fw_subcmd = FW_SCHED_SC_CONFIG;
8310         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
8311                 fw_subcmd = FW_SCHED_SC_PARAMS;
8312         else {
8313                 rc = EINVAL;
8314                 goto done;
8315         }
8316         if (p->type == SCHED_CLASS_TYPE_PACKET)
8317                 fw_type = FW_SCHED_TYPE_PKTSCHED;
8318         else {
8319                 rc = EINVAL;
8320                 goto done;
8321         }
8322
8323         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
8324                 /* Vet our parameters ..*/
8325                 if (p->u.config.minmax < 0) {
8326                         rc = EINVAL;
8327                         goto done;
8328                 }
8329
8330                 /* And pass the request to the firmware ...*/
8331                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
8332                 goto done;
8333         }
8334
8335         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
8336                 int fw_level;
8337                 int fw_mode;
8338                 int fw_rateunit;
8339                 int fw_ratemode;
8340
8341                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
8342                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
8343                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
8344                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
8345                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
8346                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
8347                 else {
8348                         rc = EINVAL;
8349                         goto done;
8350                 }
8351
8352                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
8353                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
8354                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
8355                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
8356                 else {
8357                         rc = EINVAL;
8358                         goto done;
8359                 }
8360
8361                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
8362                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
8363                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
8364                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
8365                 else {
8366                         rc = EINVAL;
8367                         goto done;
8368                 }
8369
8370                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
8371                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
8372                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
8373                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
8374                 else {
8375                         rc = EINVAL;
8376                         goto done;
8377                 }
8378
8379                 /* Vet our parameters ... */
8380                 if (!in_range(p->u.params.channel, 0, 3) ||
8381                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
8382                     !in_range(p->u.params.minrate, 0, 10000000) ||
8383                     !in_range(p->u.params.maxrate, 0, 10000000) ||
8384                     !in_range(p->u.params.weight, 0, 100)) {
8385                         rc = ERANGE;
8386                         goto done;
8387                 }
8388
8389                 /*
8390                  * Translate any unset parameters into the firmware's
8391                  * nomenclature and/or fail the call if the parameters
8392                  * are required ...
8393                  */
8394                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
8395                     p->u.params.channel < 0 || p->u.params.cl < 0) {
8396                         rc = EINVAL;
8397                         goto done;
8398                 }
8399                 if (p->u.params.minrate < 0)
8400                         p->u.params.minrate = 0;
8401                 if (p->u.params.maxrate < 0) {
8402                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8403                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8404                                 rc = EINVAL;
8405                                 goto done;
8406                         } else
8407                                 p->u.params.maxrate = 0;
8408                 }
8409                 if (p->u.params.weight < 0) {
8410                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
8411                                 rc = EINVAL;
8412                                 goto done;
8413                         } else
8414                                 p->u.params.weight = 0;
8415                 }
8416                 if (p->u.params.pktsize < 0) {
8417                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8418                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8419                                 rc = EINVAL;
8420                                 goto done;
8421                         } else
8422                                 p->u.params.pktsize = 0;
8423                 }
8424
8425                 /* See what the firmware thinks of the request ... */
8426                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
8427                     fw_rateunit, fw_ratemode, p->u.params.channel,
8428                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
8429                     p->u.params.weight, p->u.params.pktsize, 1);
8430                 goto done;
8431         }
8432
8433         rc = EINVAL;
8434 done:
8435         end_synchronized_op(sc, 0);
8436         return (rc);
8437 }
8438
8439 static int
8440 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8441 {
8442         struct port_info *pi = NULL;
8443         struct vi_info *vi;
8444         struct sge_txq *txq;
8445         uint32_t fw_mnem, fw_queue, fw_class;
8446         int i, rc;
8447
8448         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8449         if (rc)
8450                 return (rc);
8451
8452         if (!(sc->flags & FULL_INIT_DONE)) {
8453                 rc = EAGAIN;
8454                 goto done;
8455         }
8456
8457         if (p->port >= sc->params.nports) {
8458                 rc = EINVAL;
8459                 goto done;
8460         }
8461
8462         /* XXX: Only supported for the main VI. */
8463         pi = sc->port[p->port];
8464         vi = &pi->vi[0];
8465         if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
8466                 rc = EINVAL;
8467                 goto done;
8468         }
8469
8470         /*
8471          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8472          * Scheduling Class in this case).
8473          */
8474         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8475             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
8476         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
8477
8478         /*
8479          * If op.queue is non-negative, then we're only changing the scheduling
8480          * on a single specified TX queue.
8481          */
8482         if (p->queue >= 0) {
8483                 txq = &sc->sge.txq[vi->first_txq + p->queue];
8484                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8485                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8486                     &fw_class);
8487                 goto done;
8488         }
8489
8490         /*
8491          * Change the scheduling on all the TX queues for the
8492          * interface.
8493          */
8494         for_each_txq(vi, i, txq) {
8495                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8496                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8497                     &fw_class);
8498                 if (rc)
8499                         goto done;
8500         }
8501
8502         rc = 0;
8503 done:
8504         end_synchronized_op(sc, 0);
8505         return (rc);
8506 }
8507
8508 int
8509 t4_os_find_pci_capability(struct adapter *sc, int cap)
8510 {
8511         int i;
8512
8513         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
8514 }
8515
8516 int
8517 t4_os_pci_save_state(struct adapter *sc)
8518 {
8519         device_t dev;
8520         struct pci_devinfo *dinfo;
8521
8522         dev = sc->dev;
8523         dinfo = device_get_ivars(dev);
8524
8525         pci_cfg_save(dev, dinfo, 0);
8526         return (0);
8527 }
8528
8529 int
8530 t4_os_pci_restore_state(struct adapter *sc)
8531 {
8532         device_t dev;
8533         struct pci_devinfo *dinfo;
8534
8535         dev = sc->dev;
8536         dinfo = device_get_ivars(dev);
8537
8538         pci_cfg_restore(dev, dinfo);
8539         return (0);
8540 }
8541
8542 void
8543 t4_os_portmod_changed(const struct adapter *sc, int idx)
8544 {
8545         struct port_info *pi = sc->port[idx];
8546         struct vi_info *vi;
8547         struct ifnet *ifp;
8548         int v;
8549         static const char *mod_str[] = {
8550                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8551         };
8552
8553         for_each_vi(pi, v, vi) {
8554                 build_medialist(pi, &vi->media);
8555         }
8556
8557         ifp = pi->vi[0].ifp;
8558         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8559                 if_printf(ifp, "transceiver unplugged.\n");
8560         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8561                 if_printf(ifp, "unknown transceiver inserted.\n");
8562         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8563                 if_printf(ifp, "unsupported transceiver inserted.\n");
8564         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8565                 if_printf(ifp, "%s transceiver inserted.\n",
8566                     mod_str[pi->mod_type]);
8567         } else {
8568                 if_printf(ifp, "transceiver (type %d) inserted.\n",
8569                     pi->mod_type);
8570         }
8571 }
8572
8573 void
8574 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8575 {
8576         struct port_info *pi = sc->port[idx];
8577         struct vi_info *vi;
8578         struct ifnet *ifp;
8579         int v;
8580
8581         if (link_stat)
8582                 pi->linkdnrc = -1;
8583         else {
8584                 if (reason >= 0)
8585                         pi->linkdnrc = reason;
8586         }
8587         for_each_vi(pi, v, vi) {
8588                 ifp = vi->ifp;
8589                 if (ifp == NULL)
8590                         continue;
8591
8592                 if (link_stat) {
8593                         ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8594                         if_link_state_change(ifp, LINK_STATE_UP);
8595                 } else {
8596                         if_link_state_change(ifp, LINK_STATE_DOWN);
8597                 }
8598         }
8599 }
8600
8601 void
8602 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8603 {
8604         struct adapter *sc;
8605
8606         sx_slock(&t4_list_lock);
8607         SLIST_FOREACH(sc, &t4_list, link) {
8608                 /*
8609                  * func should not make any assumptions about what state sc is
8610                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8611                  */
8612                 func(sc, arg);
8613         }
8614         sx_sunlock(&t4_list_lock);
8615 }
8616
8617 static int
8618 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8619 {
8620        return (0);
8621 }
8622
8623 static int
8624 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8625 {
8626        return (0);
8627 }
8628
8629 static int
8630 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8631     struct thread *td)
8632 {
8633         int rc;
8634         struct adapter *sc = dev->si_drv1;
8635
8636         rc = priv_check(td, PRIV_DRIVER);
8637         if (rc != 0)
8638                 return (rc);
8639
8640         switch (cmd) {
8641         case CHELSIO_T4_GETREG: {
8642                 struct t4_reg *edata = (struct t4_reg *)data;
8643
8644                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8645                         return (EFAULT);
8646
8647                 if (edata->size == 4)
8648                         edata->val = t4_read_reg(sc, edata->addr);
8649                 else if (edata->size == 8)
8650                         edata->val = t4_read_reg64(sc, edata->addr);
8651                 else
8652                         return (EINVAL);
8653
8654                 break;
8655         }
8656         case CHELSIO_T4_SETREG: {
8657                 struct t4_reg *edata = (struct t4_reg *)data;
8658
8659                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8660                         return (EFAULT);
8661
8662                 if (edata->size == 4) {
8663                         if (edata->val & 0xffffffff00000000)
8664                                 return (EINVAL);
8665                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8666                 } else if (edata->size == 8)
8667                         t4_write_reg64(sc, edata->addr, edata->val);
8668                 else
8669                         return (EINVAL);
8670                 break;
8671         }
8672         case CHELSIO_T4_REGDUMP: {
8673                 struct t4_regdump *regs = (struct t4_regdump *)data;
8674                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8675                 uint8_t *buf;
8676
8677                 if (regs->len < reglen) {
8678                         regs->len = reglen; /* hint to the caller */
8679                         return (ENOBUFS);
8680                 }
8681
8682                 regs->len = reglen;
8683                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8684                 t4_get_regs(sc, regs, buf);
8685                 rc = copyout(buf, regs->data, reglen);
8686                 free(buf, M_CXGBE);
8687                 break;
8688         }
8689         case CHELSIO_T4_GET_FILTER_MODE:
8690                 rc = get_filter_mode(sc, (uint32_t *)data);
8691                 break;
8692         case CHELSIO_T4_SET_FILTER_MODE:
8693                 rc = set_filter_mode(sc, *(uint32_t *)data);
8694                 break;
8695         case CHELSIO_T4_GET_FILTER:
8696                 rc = get_filter(sc, (struct t4_filter *)data);
8697                 break;
8698         case CHELSIO_T4_SET_FILTER:
8699                 rc = set_filter(sc, (struct t4_filter *)data);
8700                 break;
8701         case CHELSIO_T4_DEL_FILTER:
8702                 rc = del_filter(sc, (struct t4_filter *)data);
8703                 break;
8704         case CHELSIO_T4_GET_SGE_CONTEXT:
8705                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8706                 break;
8707         case CHELSIO_T4_LOAD_FW:
8708                 rc = load_fw(sc, (struct t4_data *)data);
8709                 break;
8710         case CHELSIO_T4_GET_MEM:
8711                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8712                 break;
8713         case CHELSIO_T4_GET_I2C:
8714                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8715                 break;
8716         case CHELSIO_T4_CLEAR_STATS: {
8717                 int i, v;
8718                 u_int port_id = *(uint32_t *)data;
8719                 struct port_info *pi;
8720                 struct vi_info *vi;
8721
8722                 if (port_id >= sc->params.nports)
8723                         return (EINVAL);
8724                 pi = sc->port[port_id];
8725
8726                 /* MAC stats */
8727                 t4_clr_port_stats(sc, pi->tx_chan);
8728                 pi->tx_parse_error = 0;
8729                 mtx_lock(&sc->regwin_lock);
8730                 for_each_vi(pi, v, vi) {
8731                         if (vi->flags & VI_INIT_DONE)
8732                                 t4_clr_vi_stats(sc, vi->viid);
8733                 }
8734                 mtx_unlock(&sc->regwin_lock);
8735
8736                 /*
8737                  * Since this command accepts a port, clear stats for
8738                  * all VIs on this port.
8739                  */
8740                 for_each_vi(pi, v, vi) {
8741                         if (vi->flags & VI_INIT_DONE) {
8742                                 struct sge_rxq *rxq;
8743                                 struct sge_txq *txq;
8744                                 struct sge_wrq *wrq;
8745
8746                                 if (vi->flags & VI_NETMAP)
8747                                         continue;
8748
8749                                 for_each_rxq(vi, i, rxq) {
8750 #if defined(INET) || defined(INET6)
8751                                         rxq->lro.lro_queued = 0;
8752                                         rxq->lro.lro_flushed = 0;
8753 #endif
8754                                         rxq->rxcsum = 0;
8755                                         rxq->vlan_extraction = 0;
8756                                 }
8757
8758                                 for_each_txq(vi, i, txq) {
8759                                         txq->txcsum = 0;
8760                                         txq->tso_wrs = 0;
8761                                         txq->vlan_insertion = 0;
8762                                         txq->imm_wrs = 0;
8763                                         txq->sgl_wrs = 0;
8764                                         txq->txpkt_wrs = 0;
8765                                         txq->txpkts0_wrs = 0;
8766                                         txq->txpkts1_wrs = 0;
8767                                         txq->txpkts0_pkts = 0;
8768                                         txq->txpkts1_pkts = 0;
8769                                         mp_ring_reset_stats(txq->r);
8770                                 }
8771
8772 #ifdef TCP_OFFLOAD
8773                                 /* nothing to clear for each ofld_rxq */
8774
8775                                 for_each_ofld_txq(vi, i, wrq) {
8776                                         wrq->tx_wrs_direct = 0;
8777                                         wrq->tx_wrs_copied = 0;
8778                                 }
8779 #endif
8780
8781                                 if (IS_MAIN_VI(vi)) {
8782                                         wrq = &sc->sge.ctrlq[pi->port_id];
8783                                         wrq->tx_wrs_direct = 0;
8784                                         wrq->tx_wrs_copied = 0;
8785                                 }
8786                         }
8787                 }
8788                 break;
8789         }
8790         case CHELSIO_T4_SCHED_CLASS:
8791                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8792                 break;
8793         case CHELSIO_T4_SCHED_QUEUE:
8794                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8795                 break;
8796         case CHELSIO_T4_GET_TRACER:
8797                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8798                 break;
8799         case CHELSIO_T4_SET_TRACER:
8800                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8801                 break;
8802         default:
8803                 rc = EINVAL;
8804         }
8805
8806         return (rc);
8807 }
8808
8809 #ifdef TCP_OFFLOAD
8810 void
8811 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
8812 {
8813
8814         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8815         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8816                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8817                 V_HPZ3(pgsz_order[3]));
8818 }
8819
8820 static int
8821 toe_capability(struct vi_info *vi, int enable)
8822 {
8823         int rc;
8824         struct port_info *pi = vi->pi;
8825         struct adapter *sc = pi->adapter;
8826
8827         ASSERT_SYNCHRONIZED_OP(sc);
8828
8829         if (!is_offload(sc))
8830                 return (ENODEV);
8831
8832         if (enable) {
8833                 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
8834                         /* TOE is already enabled. */
8835                         return (0);
8836                 }
8837
8838                 /*
8839                  * We need the port's queues around so that we're able to send
8840                  * and receive CPLs to/from the TOE even if the ifnet for this
8841                  * port has never been UP'd administratively.
8842                  */
8843                 if (!(vi->flags & VI_INIT_DONE)) {
8844                         rc = cxgbe_init_synchronized(vi);
8845                         if (rc)
8846                                 return (rc);
8847                 }
8848                 if (!(pi->vi[0].flags & VI_INIT_DONE)) {
8849                         rc = cxgbe_init_synchronized(&pi->vi[0]);
8850                         if (rc)
8851                                 return (rc);
8852                 }
8853
8854                 if (isset(&sc->offload_map, pi->port_id)) {
8855                         /* TOE is enabled on another VI of this port. */
8856                         pi->uld_vis++;
8857                         return (0);
8858                 }
8859
8860                 if (!uld_active(sc, ULD_TOM)) {
8861                         rc = t4_activate_uld(sc, ULD_TOM);
8862                         if (rc == EAGAIN) {
8863                                 log(LOG_WARNING,
8864                                     "You must kldload t4_tom.ko before trying "
8865                                     "to enable TOE on a cxgbe interface.\n");
8866                         }
8867                         if (rc != 0)
8868                                 return (rc);
8869                         KASSERT(sc->tom_softc != NULL,
8870                             ("%s: TOM activated but softc NULL", __func__));
8871                         KASSERT(uld_active(sc, ULD_TOM),
8872                             ("%s: TOM activated but flag not set", __func__));
8873                 }
8874
8875                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8876                 if (!uld_active(sc, ULD_IWARP))
8877                         (void) t4_activate_uld(sc, ULD_IWARP);
8878                 if (!uld_active(sc, ULD_ISCSI))
8879                         (void) t4_activate_uld(sc, ULD_ISCSI);
8880
8881                 pi->uld_vis++;
8882                 setbit(&sc->offload_map, pi->port_id);
8883         } else {
8884                 pi->uld_vis--;
8885
8886                 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
8887                         return (0);
8888
8889                 KASSERT(uld_active(sc, ULD_TOM),
8890                     ("%s: TOM never initialized?", __func__));
8891                 clrbit(&sc->offload_map, pi->port_id);
8892         }
8893
8894         return (0);
8895 }
8896
8897 /*
8898  * Add an upper layer driver to the global list.
8899  */
8900 int
8901 t4_register_uld(struct uld_info *ui)
8902 {
8903         int rc = 0;
8904         struct uld_info *u;
8905
8906         sx_xlock(&t4_uld_list_lock);
8907         SLIST_FOREACH(u, &t4_uld_list, link) {
8908             if (u->uld_id == ui->uld_id) {
8909                     rc = EEXIST;
8910                     goto done;
8911             }
8912         }
8913
8914         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8915         ui->refcount = 0;
8916 done:
8917         sx_xunlock(&t4_uld_list_lock);
8918         return (rc);
8919 }
8920
8921 int
8922 t4_unregister_uld(struct uld_info *ui)
8923 {
8924         int rc = EINVAL;
8925         struct uld_info *u;
8926
8927         sx_xlock(&t4_uld_list_lock);
8928
8929         SLIST_FOREACH(u, &t4_uld_list, link) {
8930             if (u == ui) {
8931                     if (ui->refcount > 0) {
8932                             rc = EBUSY;
8933                             goto done;
8934                     }
8935
8936                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8937                     rc = 0;
8938                     goto done;
8939             }
8940         }
8941 done:
8942         sx_xunlock(&t4_uld_list_lock);
8943         return (rc);
8944 }
8945
8946 int
8947 t4_activate_uld(struct adapter *sc, int id)
8948 {
8949         int rc;
8950         struct uld_info *ui;
8951
8952         ASSERT_SYNCHRONIZED_OP(sc);
8953
8954         if (id < 0 || id > ULD_MAX)
8955                 return (EINVAL);
8956         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8957
8958         sx_slock(&t4_uld_list_lock);
8959
8960         SLIST_FOREACH(ui, &t4_uld_list, link) {
8961                 if (ui->uld_id == id) {
8962                         if (!(sc->flags & FULL_INIT_DONE)) {
8963                                 rc = adapter_full_init(sc);
8964                                 if (rc != 0)
8965                                         break;
8966                         }
8967
8968                         rc = ui->activate(sc);
8969                         if (rc == 0) {
8970                                 setbit(&sc->active_ulds, id);
8971                                 ui->refcount++;
8972                         }
8973                         break;
8974                 }
8975         }
8976
8977         sx_sunlock(&t4_uld_list_lock);
8978
8979         return (rc);
8980 }
8981
8982 int
8983 t4_deactivate_uld(struct adapter *sc, int id)
8984 {
8985         int rc;
8986         struct uld_info *ui;
8987
8988         ASSERT_SYNCHRONIZED_OP(sc);
8989
8990         if (id < 0 || id > ULD_MAX)
8991                 return (EINVAL);
8992         rc = ENXIO;
8993
8994         sx_slock(&t4_uld_list_lock);
8995
8996         SLIST_FOREACH(ui, &t4_uld_list, link) {
8997                 if (ui->uld_id == id) {
8998                         rc = ui->deactivate(sc);
8999                         if (rc == 0) {
9000                                 clrbit(&sc->active_ulds, id);
9001                                 ui->refcount--;
9002                         }
9003                         break;
9004                 }
9005         }
9006
9007         sx_sunlock(&t4_uld_list_lock);
9008
9009         return (rc);
9010 }
9011
9012 int
9013 uld_active(struct adapter *sc, int uld_id)
9014 {
9015
9016         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
9017
9018         return (isset(&sc->active_ulds, uld_id));
9019 }
9020 #endif
9021
9022 /*
9023  * Come up with reasonable defaults for some of the tunables, provided they're
9024  * not set by the user (in which case we'll use the values as is).
9025  */
9026 static void
9027 tweak_tunables(void)
9028 {
9029         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
9030
9031         if (t4_ntxq10g < 1) {
9032 #ifdef RSS
9033                 t4_ntxq10g = rss_getnumbuckets();
9034 #else
9035                 t4_ntxq10g = min(nc, NTXQ_10G);
9036 #endif
9037         }
9038
9039         if (t4_ntxq1g < 1) {
9040 #ifdef RSS
9041                 /* XXX: way too many for 1GbE? */
9042                 t4_ntxq1g = rss_getnumbuckets();
9043 #else
9044                 t4_ntxq1g = min(nc, NTXQ_1G);
9045 #endif
9046         }
9047
9048         if (t4_nrxq10g < 1) {
9049 #ifdef RSS
9050                 t4_nrxq10g = rss_getnumbuckets();
9051 #else
9052                 t4_nrxq10g = min(nc, NRXQ_10G);
9053 #endif
9054         }
9055
9056         if (t4_nrxq1g < 1) {
9057 #ifdef RSS
9058                 /* XXX: way too many for 1GbE? */
9059                 t4_nrxq1g = rss_getnumbuckets();
9060 #else
9061                 t4_nrxq1g = min(nc, NRXQ_1G);
9062 #endif
9063         }
9064
9065 #ifdef TCP_OFFLOAD
9066         if (t4_nofldtxq10g < 1)
9067                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
9068
9069         if (t4_nofldtxq1g < 1)
9070                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
9071
9072         if (t4_nofldrxq10g < 1)
9073                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
9074
9075         if (t4_nofldrxq1g < 1)
9076                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
9077
9078         if (t4_toecaps_allowed == -1)
9079                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
9080 #else
9081         if (t4_toecaps_allowed == -1)
9082                 t4_toecaps_allowed = 0;
9083 #endif
9084
9085 #ifdef DEV_NETMAP
9086         if (t4_nnmtxq10g < 1)
9087                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
9088
9089         if (t4_nnmtxq1g < 1)
9090                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
9091
9092         if (t4_nnmrxq10g < 1)
9093                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
9094
9095         if (t4_nnmrxq1g < 1)
9096                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
9097 #endif
9098
9099         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
9100                 t4_tmr_idx_10g = TMR_IDX_10G;
9101
9102         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
9103                 t4_pktc_idx_10g = PKTC_IDX_10G;
9104
9105         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
9106                 t4_tmr_idx_1g = TMR_IDX_1G;
9107
9108         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
9109                 t4_pktc_idx_1g = PKTC_IDX_1G;
9110
9111         if (t4_qsize_txq < 128)
9112                 t4_qsize_txq = 128;
9113
9114         if (t4_qsize_rxq < 128)
9115                 t4_qsize_rxq = 128;
9116         while (t4_qsize_rxq & 7)
9117                 t4_qsize_rxq++;
9118
9119         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
9120 }
9121
9122 static struct sx mlu;   /* mod load unload */
9123 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
9124
9125 static int
9126 mod_event(module_t mod, int cmd, void *arg)
9127 {
9128         int rc = 0;
9129         static int loaded = 0;
9130
9131         switch (cmd) {
9132         case MOD_LOAD:
9133                 sx_xlock(&mlu);
9134                 if (loaded++ == 0) {
9135                         t4_sge_modload();
9136                         sx_init(&t4_list_lock, "T4/T5 adapters");
9137                         SLIST_INIT(&t4_list);
9138 #ifdef TCP_OFFLOAD
9139                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
9140                         SLIST_INIT(&t4_uld_list);
9141 #endif
9142                         t4_tracer_modload();
9143                         tweak_tunables();
9144                 }
9145                 sx_xunlock(&mlu);
9146                 break;
9147
9148         case MOD_UNLOAD:
9149                 sx_xlock(&mlu);
9150                 if (--loaded == 0) {
9151                         int tries;
9152
9153                         sx_slock(&t4_list_lock);
9154                         if (!SLIST_EMPTY(&t4_list)) {
9155                                 rc = EBUSY;
9156                                 sx_sunlock(&t4_list_lock);
9157                                 goto done_unload;
9158                         }
9159 #ifdef TCP_OFFLOAD
9160                         sx_slock(&t4_uld_list_lock);
9161                         if (!SLIST_EMPTY(&t4_uld_list)) {
9162                                 rc = EBUSY;
9163                                 sx_sunlock(&t4_uld_list_lock);
9164                                 sx_sunlock(&t4_list_lock);
9165                                 goto done_unload;
9166                         }
9167 #endif
9168                         tries = 0;
9169                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
9170                                 uprintf("%ju clusters with custom free routine "
9171                                     "still is use.\n", t4_sge_extfree_refs());
9172                                 pause("t4unload", 2 * hz);
9173                         }
9174 #ifdef TCP_OFFLOAD
9175                         sx_sunlock(&t4_uld_list_lock);
9176 #endif
9177                         sx_sunlock(&t4_list_lock);
9178
9179                         if (t4_sge_extfree_refs() == 0) {
9180                                 t4_tracer_modunload();
9181 #ifdef TCP_OFFLOAD
9182                                 sx_destroy(&t4_uld_list_lock);
9183 #endif
9184                                 sx_destroy(&t4_list_lock);
9185                                 t4_sge_modunload();
9186                                 loaded = 0;
9187                         } else {
9188                                 rc = EBUSY;
9189                                 loaded++;       /* undo earlier decrement */
9190                         }
9191                 }
9192 done_unload:
9193                 sx_xunlock(&mlu);
9194                 break;
9195         }
9196
9197         return (rc);
9198 }
9199
9200 static devclass_t t4_devclass, t5_devclass;
9201 static devclass_t cxgbe_devclass, cxl_devclass;
9202 static devclass_t vcxgbe_devclass, vcxl_devclass;
9203
9204 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
9205 MODULE_VERSION(t4nex, 1);
9206 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
9207 #ifdef DEV_NETMAP
9208 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
9209 #endif /* DEV_NETMAP */
9210
9211
9212 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
9213 MODULE_VERSION(t5nex, 1);
9214 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
9215 #ifdef DEV_NETMAP
9216 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
9217 #endif /* DEV_NETMAP */
9218
9219 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
9220 MODULE_VERSION(cxgbe, 1);
9221
9222 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
9223 MODULE_VERSION(cxl, 1);
9224
9225 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
9226 MODULE_VERSION(vcxgbe, 1);
9227
9228 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
9229 MODULE_VERSION(vcxl, 1);