sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #ifdef RSS
  61 #include <net/rss_config.h>
  62 #endif
  63 #if defined(__i386__) || defined(__amd64__)
  64 #include <vm/vm.h>
  65 #include <vm/pmap.h>
  66 #endif
  67
  68 #include "common/common.h"
  69 #include "common/t4_msg.h"
  70 #include "common/t4_regs.h"
  71 #include "common/t4_regs_values.h"
  72 #include "t4_ioctl.h"
  73 #include "t4_l2t.h"
  74 #include "t4_mp_ring.h"
  75
  76 /* T4 bus driver interface */
  77 static int t4_probe(device_t);
  78 static int t4_attach(device_t);
  79 static int t4_detach(device_t);
  80 static device_method_t t4_methods[] = {
  81         DEVMETHOD(device_probe,         t4_probe),
  82         DEVMETHOD(device_attach,        t4_attach),
  83         DEVMETHOD(device_detach,        t4_detach),
  84
  85         DEVMETHOD_END
  86 };
  87 static driver_t t4_driver = {
  88         "t4nex",
  89         t4_methods,
  90         sizeof(struct adapter)
  91 };
  92
  93
  94 /* T4 port (cxgbe) interface */
  95 static int cxgbe_probe(device_t);
  96 static int cxgbe_attach(device_t);
  97 static int cxgbe_detach(device_t);
  98 static device_method_t cxgbe_methods[] = {
  99         DEVMETHOD(device_probe,         cxgbe_probe),
 100         DEVMETHOD(device_attach,        cxgbe_attach),
 101         DEVMETHOD(device_detach,        cxgbe_detach),
 102         { 0, 0 }
 103 };
 104 static driver_t cxgbe_driver = {
 105         "cxgbe",
 106         cxgbe_methods,
 107         sizeof(struct port_info)
 108 };
 109
 110 /* T4 VI (vcxgbe) interface */
 111 static int vcxgbe_probe(device_t);
 112 static int vcxgbe_attach(device_t);
 113 static int vcxgbe_detach(device_t);
 114 static device_method_t vcxgbe_methods[] = {
 115         DEVMETHOD(device_probe,         vcxgbe_probe),
 116         DEVMETHOD(device_attach,        vcxgbe_attach),
 117         DEVMETHOD(device_detach,        vcxgbe_detach),
 118         { 0, 0 }
 119 };
 120 static driver_t vcxgbe_driver = {
 121         "vcxgbe",
 122         vcxgbe_methods,
 123         sizeof(struct vi_info)
 124 };
 125
 126 static d_ioctl_t t4_ioctl;
 127 static d_open_t t4_open;
 128 static d_close_t t4_close;
 129
 130 static struct cdevsw t4_cdevsw = {
 131        .d_version = D_VERSION,
 132        .d_flags = 0,
 133        .d_open = t4_open,
 134        .d_close = t4_close,
 135        .d_ioctl = t4_ioctl,
 136        .d_name = "t4nex",
 137 };
 138
 139 /* T5 bus driver interface */
 140 static int t5_probe(device_t);
 141 static device_method_t t5_methods[] = {
 142         DEVMETHOD(device_probe,         t5_probe),
 143         DEVMETHOD(device_attach,        t4_attach),
 144         DEVMETHOD(device_detach,        t4_detach),
 145
 146         DEVMETHOD_END
 147 };
 148 static driver_t t5_driver = {
 149         "t5nex",
 150         t5_methods,
 151         sizeof(struct adapter)
 152 };
 153
 154
 155 /* T5 port (cxl) interface */
 156 static driver_t cxl_driver = {
 157         "cxl",
 158         cxgbe_methods,
 159         sizeof(struct port_info)
 160 };
 161
 162 /* T5 VI (vcxl) interface */
 163 static driver_t vcxl_driver = {
 164         "vcxl",
 165         vcxgbe_methods,
 166         sizeof(struct vi_info)
 167 };
 168
 169 static struct cdevsw t5_cdevsw = {
 170        .d_version = D_VERSION,
 171        .d_flags = 0,
 172        .d_open = t4_open,
 173        .d_close = t4_close,
 174        .d_ioctl = t4_ioctl,
 175        .d_name = "t5nex",
 176 };
 177
 178 /* ifnet + media interface */
 179 static void cxgbe_init(void *);
 180 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 181 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 182 static void cxgbe_qflush(struct ifnet *);
 183 static int cxgbe_media_change(struct ifnet *);
 184 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 185
 186 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 187
 188 /*
 189  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 190  * then ADAPTER_LOCK, then t4_uld_list_lock.
 191  */
 192 static struct sx t4_list_lock;
 193 SLIST_HEAD(, adapter) t4_list;
 194 #ifdef TCP_OFFLOAD
 195 static struct sx t4_uld_list_lock;
 196 SLIST_HEAD(, uld_info) t4_uld_list;
 197 #endif
 198
 199 /*
 200  * Tunables.  See tweak_tunables() too.
 201  *
 202  * Each tunable is set to a default value here if it's known at compile-time.
 203  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 204  * provide a reasonable default when the driver is loaded.
 205  *
 206  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 207  * T5 are under hw.cxl.
 208  */
 209
 210 /*
 211  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 212  */
 213 #define NTXQ_10G 16
 214 static int t4_ntxq10g = -1;
 215 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 216
 217 #define NRXQ_10G 8
 218 static int t4_nrxq10g = -1;
 219 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 220
 221 #define NTXQ_1G 4
 222 static int t4_ntxq1g = -1;
 223 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 224
 225 #define NRXQ_1G 2
 226 static int t4_nrxq1g = -1;
 227 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 228
 229 #define NTXQ_VI 1
 230 static int t4_ntxq_vi = -1;
 231 TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi);
 232
 233 #define NRXQ_VI 1
 234 static int t4_nrxq_vi = -1;
 235 TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi);
 236
 237 static int t4_rsrv_noflowq = 0;
 238 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 239
 240 #ifdef TCP_OFFLOAD
 241 #define NOFLDTXQ_10G 8
 242 static int t4_nofldtxq10g = -1;
 243 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 244
 245 #define NOFLDRXQ_10G 2
 246 static int t4_nofldrxq10g = -1;
 247 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 248
 249 #define NOFLDTXQ_1G 2
 250 static int t4_nofldtxq1g = -1;
 251 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 252
 253 #define NOFLDRXQ_1G 1
 254 static int t4_nofldrxq1g = -1;
 255 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 256
 257 #define NOFLDTXQ_VI 1
 258 static int t4_nofldtxq_vi = -1;
 259 TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi);
 260
 261 #define NOFLDRXQ_VI 1
 262 static int t4_nofldrxq_vi = -1;
 263 TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi);
 264 #endif
 265
 266 #ifdef DEV_NETMAP
 267 #define NNMTXQ_VI 2
 268 static int t4_nnmtxq_vi = -1;
 269 TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi);
 270
 271 #define NNMRXQ_VI 2
 272 static int t4_nnmrxq_vi = -1;
 273 TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi);
 274 #endif
 275
 276 /*
 277  * Holdoff parameters for 10G and 1G ports.
 278  */
 279 #define TMR_IDX_10G 1
 280 static int t4_tmr_idx_10g = TMR_IDX_10G;
 281 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 282
 283 #define PKTC_IDX_10G (-1)
 284 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 285 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 286
 287 #define TMR_IDX_1G 1
 288 static int t4_tmr_idx_1g = TMR_IDX_1G;
 289 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 290
 291 #define PKTC_IDX_1G (-1)
 292 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 293 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 294
 295 /*
 296  * Size (# of entries) of each tx and rx queue.
 297  */
 298 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 299 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 300
 301 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 302 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 303
 304 /*
 305  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 306  */
 307 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 308 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 309
 310 /*
 311  * Configuration file.
 312  */
 313 #define DEFAULT_CF      "default"
 314 #define FLASH_CF        "flash"
 315 #define UWIRE_CF        "uwire"
 316 #define FPGA_CF         "fpga"
 317 static char t4_cfg_file[32] = DEFAULT_CF;
 318 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 319
 320 /*
 321  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 322  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 323  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 324  *            mark or when signalled to do so, 0 to never emit PAUSE.
 325  */
 326 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 327 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 328
 329 /*
 330  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 331  * encouraged respectively).
 332  */
 333 static unsigned int t4_fw_install = 1;
 334 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 335
 336 /*
 337  * ASIC features that will be used.  Disable the ones you don't want so that the
 338  * chip resources aren't wasted on features that will not be used.
 339  */
 340 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 341 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 342
 343 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 344 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 345
 346 static int t4_toecaps_allowed = -1;
 347 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 348
 349 static int t4_rdmacaps_allowed = 0;
 350 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 351
 352 static int t4_iscsicaps_allowed = 0;
 353 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 354
 355 static int t4_fcoecaps_allowed = 0;
 356 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 357
 358 static int t5_write_combine = 0;
 359 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 360
 361 static int t4_num_vis = 1;
 362 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
 363
 364 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */
 365 static int vi_mac_funcs[] = {
 366         FW_VI_FUNC_OFLD,
 367         FW_VI_FUNC_IWARP,
 368         FW_VI_FUNC_OPENISCSI,
 369         FW_VI_FUNC_OPENFCOE,
 370         FW_VI_FUNC_FOISCSI,
 371         FW_VI_FUNC_FOFCOE,
 372 };
 373
 374 struct intrs_and_queues {
 375         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 376         uint16_t nirq;          /* Total # of vectors */
 377         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 378         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 379         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 380         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 381         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 382         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 383         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 384         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 385         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 386         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 387         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 388
 389         /* The vcxgbe/vcxl interfaces use these and not the ones above. */
 390         uint16_t ntxq_vi;       /* # of NIC txq's */
 391         uint16_t nrxq_vi;       /* # of NIC rxq's */
 392         uint16_t nofldtxq_vi;   /* # of TOE txq's */
 393         uint16_t nofldrxq_vi;   /* # of TOE rxq's */
 394         uint16_t nnmtxq_vi;     /* # of netmap txq's */
 395         uint16_t nnmrxq_vi;     /* # of netmap rxq's */
 396 };
 397
 398 struct filter_entry {
 399         uint32_t valid:1;       /* filter allocated and valid */
 400         uint32_t locked:1;      /* filter is administratively locked */
 401         uint32_t pending:1;     /* filter action is pending firmware reply */
 402         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 403         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 404
 405         struct t4_filter_specification fs;
 406 };
 407
 408 static int map_bars_0_and_4(struct adapter *);
 409 static int map_bar_2(struct adapter *);
 410 static void setup_memwin(struct adapter *);
 411 static int validate_mem_range(struct adapter *, uint32_t, int);
 412 static int fwmtype_to_hwmtype(int);
 413 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 414     uint32_t *);
 415 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 416 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 417 static int cfg_itype_and_nqueues(struct adapter *, int, int, int,
 418     struct intrs_and_queues *);
 419 static int prep_firmware(struct adapter *);
 420 static int partition_resources(struct adapter *, const struct firmware *,
 421     const char *);
 422 static int get_params__pre_init(struct adapter *);
 423 static int get_params__post_init(struct adapter *);
 424 static int set_params__post_init(struct adapter *);
 425 static void t4_set_desc(struct adapter *);
 426 static void build_medialist(struct port_info *, struct ifmedia *);
 427 static int cxgbe_init_synchronized(struct vi_info *);
 428 static int cxgbe_uninit_synchronized(struct vi_info *);
 429 static int setup_intr_handlers(struct adapter *);
 430 static void quiesce_txq(struct adapter *, struct sge_txq *);
 431 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 432 static void quiesce_iq(struct adapter *, struct sge_iq *);
 433 static void quiesce_fl(struct adapter *, struct sge_fl *);
 434 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 435     driver_intr_t *, void *, char *);
 436 static int t4_free_irq(struct adapter *, struct irq *);
 437 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 438     unsigned int);
 439 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 440 static void vi_refresh_stats(struct adapter *, struct vi_info *);
 441 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 442 static void cxgbe_tick(void *);
 443 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 444 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 445     struct mbuf *);
 446 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 447 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 448 static void t4_sysctls(struct adapter *);
 449 static void cxgbe_sysctls(struct port_info *);
 450 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 451 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 452 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 453 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 454 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 455 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 456 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 457 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 458 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 459 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 460 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 461 #ifdef SBUF_DRAIN
 462 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 463 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 464 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 465 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 466 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 467 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 468 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 469 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 470 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 471 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 472 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 473 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 474 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 475 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 476 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 477 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 478 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 479 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 480 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 481 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 482 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 483 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 484 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 485 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 486 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 487 #endif
 488 static uint32_t fconf_to_mode(uint32_t);
 489 static uint32_t mode_to_fconf(uint32_t);
 490 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 491 static int get_filter_mode(struct adapter *, uint32_t *);
 492 static int set_filter_mode(struct adapter *, uint32_t);
 493 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 494 static int get_filter(struct adapter *, struct t4_filter *);
 495 static int set_filter(struct adapter *, struct t4_filter *);
 496 static int del_filter(struct adapter *, struct t4_filter *);
 497 static void clear_filter(struct filter_entry *);
 498 static int set_filter_wr(struct adapter *, int);
 499 static int del_filter_wr(struct adapter *, int);
 500 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 501 static int load_fw(struct adapter *, struct t4_data *);
 502 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 503 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 504 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 505 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 506 #ifdef TCP_OFFLOAD
 507 static int toe_capability(struct vi_info *, int);
 508 #endif
 509 static int mod_event(module_t, int, void *);
 510
 511 struct {
 512         uint16_t device;
 513         char *desc;
 514 } t4_pciids[] = {
 515         {0xa000, "Chelsio Terminator 4 FPGA"},
 516         {0x4400, "Chelsio T440-dbg"},
 517         {0x4401, "Chelsio T420-CR"},
 518         {0x4402, "Chelsio T422-CR"},
 519         {0x4403, "Chelsio T440-CR"},
 520         {0x4404, "Chelsio T420-BCH"},
 521         {0x4405, "Chelsio T440-BCH"},
 522         {0x4406, "Chelsio T440-CH"},
 523         {0x4407, "Chelsio T420-SO"},
 524         {0x4408, "Chelsio T420-CX"},
 525         {0x4409, "Chelsio T420-BT"},
 526         {0x440a, "Chelsio T404-BT"},
 527         {0x440e, "Chelsio T440-LP-CR"},
 528 }, t5_pciids[] = {
 529         {0xb000, "Chelsio Terminator 5 FPGA"},
 530         {0x5400, "Chelsio T580-dbg"},
 531         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 532         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 533         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 534         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 535         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 536         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 537         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 538         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 539         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 540         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 541         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 542         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 543         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 544 #ifdef notyet
 545         {0x5404,  "Chelsio T520-BCH"},
 546         {0x5405,  "Chelsio T540-BCH"},
 547         {0x5406,  "Chelsio T540-CH"},
 548         {0x5408,  "Chelsio T520-CX"},
 549         {0x540b,  "Chelsio B520-SR"},
 550         {0x540c,  "Chelsio B504-BT"},
 551         {0x540f,  "Chelsio Amsterdam"},
 552         {0x5413,  "Chelsio T580-CHR"},
 553 #endif
 554 };
 555
 556 #ifdef TCP_OFFLOAD
 557 /*
 558  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 559  * exactly the same for both rxq and ofld_rxq.
 560  */
 561 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 562 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 563 #endif
 564
 565 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 566 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 567 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 568
 569 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 570
 571 static int
 572 t4_probe(device_t dev)
 573 {
 574         int i;
 575         uint16_t v = pci_get_vendor(dev);
 576         uint16_t d = pci_get_device(dev);
 577         uint8_t f = pci_get_function(dev);
 578
 579         if (v != PCI_VENDOR_ID_CHELSIO)
 580                 return (ENXIO);
 581
 582         /* Attach only to PF0 of the FPGA */
 583         if (d == 0xa000 && f != 0)
 584                 return (ENXIO);
 585
 586         for (i = 0; i < nitems(t4_pciids); i++) {
 587                 if (d == t4_pciids[i].device) {
 588                         device_set_desc(dev, t4_pciids[i].desc);
 589                         return (BUS_PROBE_DEFAULT);
 590                 }
 591         }
 592
 593         return (ENXIO);
 594 }
 595
 596 static int
 597 t5_probe(device_t dev)
 598 {
 599         int i;
 600         uint16_t v = pci_get_vendor(dev);
 601         uint16_t d = pci_get_device(dev);
 602         uint8_t f = pci_get_function(dev);
 603
 604         if (v != PCI_VENDOR_ID_CHELSIO)
 605                 return (ENXIO);
 606
 607         /* Attach only to PF0 of the FPGA */
 608         if (d == 0xb000 && f != 0)
 609                 return (ENXIO);
 610
 611         for (i = 0; i < nitems(t5_pciids); i++) {
 612                 if (d == t5_pciids[i].device) {
 613                         device_set_desc(dev, t5_pciids[i].desc);
 614                         return (BUS_PROBE_DEFAULT);
 615                 }
 616         }
 617
 618         return (ENXIO);
 619 }
 620
 621 static void
 622 t5_attribute_workaround(device_t dev)
 623 {
 624         device_t root_port;
 625         uint32_t v;
 626
 627         /*
 628          * The T5 chips do not properly echo the No Snoop and Relaxed
 629          * Ordering attributes when replying to a TLP from a Root
 630          * Port.  As a workaround, find the parent Root Port and
 631          * disable No Snoop and Relaxed Ordering.  Note that this
 632          * affects all devices under this root port.
 633          */
 634         root_port = pci_find_pcie_root_port(dev);
 635         if (root_port == NULL) {
 636                 device_printf(dev, "Unable to find parent root port\n");
 637                 return;
 638         }
 639
 640         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 641             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 642         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 643             0)
 644                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 645                     device_get_nameunit(root_port));
 646 }
 647
 648 static int
 649 t4_attach(device_t dev)
 650 {
 651         struct adapter *sc;
 652         int rc = 0, i, j, n10g, n1g, rqidx, tqidx;
 653         struct intrs_and_queues iaq;
 654         struct sge *s;
 655 #ifdef TCP_OFFLOAD
 656         int ofld_rqidx, ofld_tqidx;
 657 #endif
 658 #ifdef DEV_NETMAP
 659         int nm_rqidx, nm_tqidx;
 660 #endif
 661         int num_vis;
 662
 663         sc = device_get_softc(dev);
 664         sc->dev = dev;
 665         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 666
 667         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 668                 t5_attribute_workaround(dev);
 669         pci_enable_busmaster(dev);
 670         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 671                 uint32_t v;
 672
 673                 pci_set_max_read_req(dev, 4096);
 674                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 675                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 676                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 677
 678                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 679         }
 680
 681         sc->traceq = -1;
 682         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 683         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 684             device_get_nameunit(dev));
 685
 686         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 687             device_get_nameunit(dev));
 688         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 689         sx_xlock(&t4_list_lock);
 690         SLIST_INSERT_HEAD(&t4_list, sc, link);
 691         sx_xunlock(&t4_list_lock);
 692
 693         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 694         TAILQ_INIT(&sc->sfl);
 695         callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
 696
 697         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 698
 699         rc = map_bars_0_and_4(sc);
 700         if (rc != 0)
 701                 goto done; /* error message displayed already */
 702
 703         /*
 704          * This is the real PF# to which we're attaching.  Works from within PCI
 705          * passthrough environments too, where pci_get_function() could return a
 706          * different PF# depending on the passthrough configuration.  We need to
 707          * use the real PF# in all our communication with the firmware.
 708          */
 709         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 710         sc->mbox = sc->pf;
 711
 712         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 713         sc->an_handler = an_not_handled;
 714         for (i = 0; i < nitems(sc->cpl_handler); i++)
 715                 sc->cpl_handler[i] = cpl_not_handled;
 716         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 717                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 718         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 719         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 720         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 721         t4_init_sge_cpl_handlers(sc);
 722
 723         /* Prepare the adapter for operation */
 724         rc = -t4_prep_adapter(sc);
 725         if (rc != 0) {
 726                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 727                 goto done;
 728         }
 729
 730         /*
 731          * Do this really early, with the memory windows set up even before the
 732          * character device.  The userland tool's register i/o and mem read
 733          * will work even in "recovery mode".
 734          */
 735         setup_memwin(sc);
 736         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 737             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 738             device_get_nameunit(dev));
 739         if (sc->cdev == NULL)
 740                 device_printf(dev, "failed to create nexus char device.\n");
 741         else
 742                 sc->cdev->si_drv1 = sc;
 743
 744         /* Go no further if recovery mode has been requested. */
 745         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 746                 device_printf(dev, "recovery mode.\n");
 747                 goto done;
 748         }
 749
 750 #if defined(__i386__)
 751         if ((cpu_feature & CPUID_CX8) == 0) {
 752                 device_printf(dev, "64 bit atomics not available.\n");
 753                 rc = ENOTSUP;
 754                 goto done;
 755         }
 756 #endif
 757
 758         /* Prepare the firmware for operation */
 759         rc = prep_firmware(sc);
 760         if (rc != 0)
 761                 goto done; /* error message displayed already */
 762
 763         rc = get_params__post_init(sc);
 764         if (rc != 0)
 765                 goto done; /* error message displayed already */
 766
 767         rc = set_params__post_init(sc);
 768         if (rc != 0)
 769                 goto done; /* error message displayed already */
 770
 771         rc = map_bar_2(sc);
 772         if (rc != 0)
 773                 goto done; /* error message displayed already */
 774
 775         rc = t4_create_dma_tag(sc);
 776         if (rc != 0)
 777                 goto done; /* error message displayed already */
 778
 779         /*
 780          * Number of VIs to create per-port.  The first VI is the "main" regular
 781          * VI for the port.  The rest are additional virtual interfaces on the
 782          * same physical port.  Note that the main VI does not have native
 783          * netmap support but the extra VIs do.
 784          *
 785          * Limit the number of VIs per port to the number of available
 786          * MAC addresses per port.
 787          */
 788         if (t4_num_vis >= 1)
 789                 num_vis = t4_num_vis;
 790         else
 791                 num_vis = 1;
 792         if (num_vis > nitems(vi_mac_funcs)) {
 793                 num_vis = nitems(vi_mac_funcs);
 794                 device_printf(dev, "Number of VIs limited to %d\n", num_vis);
 795         }
 796
 797         /*
 798          * First pass over all the ports - allocate VIs and initialize some
 799          * basic parameters like mac address, port type, etc.  We also figure
 800          * out whether a port is 10G or 1G and use that information when
 801          * calculating how many interrupts to attempt to allocate.
 802          */
 803         n10g = n1g = 0;
 804         for_each_port(sc, i) {
 805                 struct port_info *pi;
 806
 807                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 808                 sc->port[i] = pi;
 809
 810                 /* These must be set before t4_port_init */
 811                 pi->adapter = sc;
 812                 pi->port_id = i;
 813                 /*
 814                  * XXX: vi[0] is special so we can't delay this allocation until
 815                  * pi->nvi's final value is known.
 816                  */
 817                 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE,
 818                     M_ZERO | M_WAITOK);
 819
 820                 /*
 821                  * Allocate the "main" VI and initialize parameters
 822                  * like mac addr.
 823                  */
 824                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 825                 if (rc != 0) {
 826                         device_printf(dev, "unable to initialize port %d: %d\n",
 827                             i, rc);
 828                         free(pi->vi, M_CXGBE);
 829                         free(pi, M_CXGBE);
 830                         sc->port[i] = NULL;
 831                         goto done;
 832                 }
 833
 834                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 835                 pi->link_cfg.requested_fc |= t4_pause_settings;
 836                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 837                 pi->link_cfg.fc |= t4_pause_settings;
 838
 839                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 840                 if (rc != 0) {
 841                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 842                         free(pi->vi, M_CXGBE);
 843                         free(pi, M_CXGBE);
 844                         sc->port[i] = NULL;
 845                         goto done;
 846                 }
 847
 848                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 849                     device_get_nameunit(dev), i);
 850                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 851                 sc->chan_map[pi->tx_chan] = i;
 852
 853                 if (is_10G_port(pi) || is_40G_port(pi)) {
 854                         n10g++;
 855                 } else {
 856                         n1g++;
 857                 }
 858
 859                 pi->linkdnrc = -1;
 860
 861                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 862                 if (pi->dev == NULL) {
 863                         device_printf(dev,
 864                             "failed to add device for port %d.\n", i);
 865                         rc = ENXIO;
 866                         goto done;
 867                 }
 868                 pi->vi[0].dev = pi->dev;
 869                 device_set_softc(pi->dev, pi);
 870         }
 871
 872         /*
 873          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 874          */
 875         rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq);
 876         if (rc != 0)
 877                 goto done; /* error message displayed already */
 878         if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0)
 879                 num_vis = 1;
 880
 881         sc->intr_type = iaq.intr_type;
 882         sc->intr_count = iaq.nirq;
 883
 884         s = &sc->sge;
 885         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 886         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 887         if (num_vis > 1) {
 888                 s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi;
 889                 s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi;
 890         }
 891         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 892         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 893         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 894 #ifdef TCP_OFFLOAD
 895         if (is_offload(sc)) {
 896                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 897                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 898                 if (num_vis > 1) {
 899                         s->nofldrxq += (n10g + n1g) * (num_vis - 1) *
 900                             iaq.nofldrxq_vi;
 901                         s->nofldtxq += (n10g + n1g) * (num_vis - 1) *
 902                             iaq.nofldtxq_vi;
 903                 }
 904                 s->neq += s->nofldtxq + s->nofldrxq;
 905                 s->niq += s->nofldrxq;
 906
 907                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 908                     M_CXGBE, M_ZERO | M_WAITOK);
 909                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 910                     M_CXGBE, M_ZERO | M_WAITOK);
 911         }
 912 #endif
 913 #ifdef DEV_NETMAP
 914         if (num_vis > 1) {
 915                 s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi;
 916                 s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi;
 917         }
 918         s->neq += s->nnmtxq + s->nnmrxq;
 919         s->niq += s->nnmrxq;
 920
 921         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 922             M_CXGBE, M_ZERO | M_WAITOK);
 923         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 924             M_CXGBE, M_ZERO | M_WAITOK);
 925 #endif
 926
 927         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 928             M_ZERO | M_WAITOK);
 929         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 930             M_ZERO | M_WAITOK);
 931         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 932             M_ZERO | M_WAITOK);
 933         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 934             M_ZERO | M_WAITOK);
 935         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 936             M_ZERO | M_WAITOK);
 937
 938         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 939             M_ZERO | M_WAITOK);
 940
 941         t4_init_l2t(sc, M_WAITOK);
 942
 943         /*
 944          * Second pass over the ports.  This time we know the number of rx and
 945          * tx queues that each port should get.
 946          */
 947         rqidx = tqidx = 0;
 948 #ifdef TCP_OFFLOAD
 949         ofld_rqidx = ofld_tqidx = 0;
 950 #endif
 951 #ifdef DEV_NETMAP
 952         nm_rqidx = nm_tqidx = 0;
 953 #endif
 954         for_each_port(sc, i) {
 955                 struct port_info *pi = sc->port[i];
 956                 struct vi_info *vi;
 957
 958                 if (pi == NULL)
 959                         continue;
 960
 961                 pi->nvi = num_vis;
 962                 for_each_vi(pi, j, vi) {
 963                         vi->pi = pi;
 964                         vi->qsize_rxq = t4_qsize_rxq;
 965                         vi->qsize_txq = t4_qsize_txq;
 966
 967                         vi->first_rxq = rqidx;
 968                         vi->first_txq = tqidx;
 969                         if (is_10G_port(pi) || is_40G_port(pi)) {
 970                                 vi->tmr_idx = t4_tmr_idx_10g;
 971                                 vi->pktc_idx = t4_pktc_idx_10g;
 972                                 vi->flags |= iaq.intr_flags_10g & INTR_RXQ;
 973                                 vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi;
 974                                 vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi;
 975                         } else {
 976                                 vi->tmr_idx = t4_tmr_idx_1g;
 977                                 vi->pktc_idx = t4_pktc_idx_1g;
 978                                 vi->flags |= iaq.intr_flags_1g & INTR_RXQ;
 979                                 vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi;
 980                                 vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi;
 981                         }
 982                         rqidx += vi->nrxq;
 983                         tqidx += vi->ntxq;
 984
 985                         if (j == 0 && vi->ntxq > 1)
 986                                 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 987                         else
 988                                 vi->rsrv_noflowq = 0;
 989
 990 #ifdef TCP_OFFLOAD
 991                         vi->first_ofld_rxq = ofld_rqidx;
 992                         vi->first_ofld_txq = ofld_tqidx;
 993                         if (is_10G_port(pi) || is_40G_port(pi)) {
 994                                 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ;
 995                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g :
 996                                     iaq.nofldrxq_vi;
 997                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g :
 998                                     iaq.nofldtxq_vi;
 999                         } else {
1000                                 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ;
1001                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g :
1002                                     iaq.nofldrxq_vi;
1003                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g :
1004                                     iaq.nofldtxq_vi;
1005                         }
1006                         ofld_rqidx += vi->nofldrxq;
1007                         ofld_tqidx += vi->nofldtxq;
1008 #endif
1009 #ifdef DEV_NETMAP
1010                         if (j > 0) {
1011                                 vi->first_nm_rxq = nm_rqidx;
1012                                 vi->first_nm_txq = nm_tqidx;
1013                                 vi->nnmrxq = iaq.nnmrxq_vi;
1014                                 vi->nnmtxq = iaq.nnmtxq_vi;
1015                                 nm_rqidx += vi->nnmrxq;
1016                                 nm_tqidx += vi->nnmtxq;
1017                         }
1018 #endif
1019                 }
1020         }
1021
1022         rc = setup_intr_handlers(sc);
1023         if (rc != 0) {
1024                 device_printf(dev,
1025                     "failed to setup interrupt handlers: %d\n", rc);
1026                 goto done;
1027         }
1028
1029         rc = bus_generic_attach(dev);
1030         if (rc != 0) {
1031                 device_printf(dev,
1032                     "failed to attach all child ports: %d\n", rc);
1033                 goto done;
1034         }
1035
1036         device_printf(dev,
1037             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1038             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1039             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1040             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1041             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1042
1043         t4_set_desc(sc);
1044
1045 done:
1046         if (rc != 0 && sc->cdev) {
1047                 /* cdev was created and so cxgbetool works; recover that way. */
1048                 device_printf(dev,
1049                     "error during attach, adapter is now in recovery mode.\n");
1050                 rc = 0;
1051         }
1052
1053         if (rc != 0)
1054                 t4_detach(dev);
1055         else
1056                 t4_sysctls(sc);
1057
1058         return (rc);
1059 }
1060
1061 /*
1062  * Idempotent
1063  */
1064 static int
1065 t4_detach(device_t dev)
1066 {
1067         struct adapter *sc;
1068         struct port_info *pi;
1069         int i, rc;
1070
1071         sc = device_get_softc(dev);
1072
1073         if (sc->flags & FULL_INIT_DONE)
1074                 t4_intr_disable(sc);
1075
1076         if (sc->cdev) {
1077                 destroy_dev(sc->cdev);
1078                 sc->cdev = NULL;
1079         }
1080
1081         rc = bus_generic_detach(dev);
1082         if (rc) {
1083                 device_printf(dev,
1084                     "failed to detach child devices: %d\n", rc);
1085                 return (rc);
1086         }
1087
1088         for (i = 0; i < sc->intr_count; i++)
1089                 t4_free_irq(sc, &sc->irq[i]);
1090
1091         for (i = 0; i < MAX_NPORTS; i++) {
1092                 pi = sc->port[i];
1093                 if (pi) {
1094                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1095                         if (pi->dev)
1096                                 device_delete_child(dev, pi->dev);
1097
1098                         mtx_destroy(&pi->pi_lock);
1099                         free(pi->vi, M_CXGBE);
1100                         free(pi, M_CXGBE);
1101                 }
1102         }
1103
1104         if (sc->flags & FULL_INIT_DONE)
1105                 adapter_full_uninit(sc);
1106
1107         if (sc->flags & FW_OK)
1108                 t4_fw_bye(sc, sc->mbox);
1109
1110         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1111                 pci_release_msi(dev);
1112
1113         if (sc->regs_res)
1114                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1115                     sc->regs_res);
1116
1117         if (sc->udbs_res)
1118                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1119                     sc->udbs_res);
1120
1121         if (sc->msix_res)
1122                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1123                     sc->msix_res);
1124
1125         if (sc->l2t)
1126                 t4_free_l2t(sc->l2t);
1127
1128 #ifdef TCP_OFFLOAD
1129         free(sc->sge.ofld_rxq, M_CXGBE);
1130         free(sc->sge.ofld_txq, M_CXGBE);
1131 #endif
1132 #ifdef DEV_NETMAP
1133         free(sc->sge.nm_rxq, M_CXGBE);
1134         free(sc->sge.nm_txq, M_CXGBE);
1135 #endif
1136         free(sc->irq, M_CXGBE);
1137         free(sc->sge.rxq, M_CXGBE);
1138         free(sc->sge.txq, M_CXGBE);
1139         free(sc->sge.ctrlq, M_CXGBE);
1140         free(sc->sge.iqmap, M_CXGBE);
1141         free(sc->sge.eqmap, M_CXGBE);
1142         free(sc->tids.ftid_tab, M_CXGBE);
1143         t4_destroy_dma_tag(sc);
1144         if (mtx_initialized(&sc->sc_lock)) {
1145                 sx_xlock(&t4_list_lock);
1146                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1147                 sx_xunlock(&t4_list_lock);
1148                 mtx_destroy(&sc->sc_lock);
1149         }
1150
1151         callout_drain(&sc->sfl_callout);
1152         if (mtx_initialized(&sc->tids.ftid_lock))
1153                 mtx_destroy(&sc->tids.ftid_lock);
1154         if (mtx_initialized(&sc->sfl_lock))
1155                 mtx_destroy(&sc->sfl_lock);
1156         if (mtx_initialized(&sc->ifp_lock))
1157                 mtx_destroy(&sc->ifp_lock);
1158         if (mtx_initialized(&sc->regwin_lock))
1159                 mtx_destroy(&sc->regwin_lock);
1160
1161         bzero(sc, sizeof(*sc));
1162
1163         return (0);
1164 }
1165
1166 static int
1167 cxgbe_probe(device_t dev)
1168 {
1169         char buf[128];
1170         struct port_info *pi = device_get_softc(dev);
1171
1172         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1173         device_set_desc_copy(dev, buf);
1174
1175         return (BUS_PROBE_DEFAULT);
1176 }
1177
1178 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1179     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1180     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1181 #define T4_CAP_ENABLE (T4_CAP)
1182
1183 static int
1184 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1185 {
1186         struct ifnet *ifp;
1187         struct sbuf *sb;
1188
1189         vi->xact_addr_filt = -1;
1190         callout_init(&vi->tick, 1);
1191
1192         /* Allocate an ifnet and set it up */
1193         ifp = if_alloc(IFT_ETHER);
1194         if (ifp == NULL) {
1195                 device_printf(dev, "Cannot allocate ifnet\n");
1196                 return (ENOMEM);
1197         }
1198         vi->ifp = ifp;
1199         ifp->if_softc = vi;
1200
1201         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1202         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1203
1204         ifp->if_init = cxgbe_init;
1205         ifp->if_ioctl = cxgbe_ioctl;
1206         ifp->if_transmit = cxgbe_transmit;
1207         ifp->if_qflush = cxgbe_qflush;
1208
1209         ifp->if_capabilities = T4_CAP;
1210 #ifdef TCP_OFFLOAD
1211         if (vi->nofldrxq != 0)
1212                 ifp->if_capabilities |= IFCAP_TOE;
1213 #endif
1214         ifp->if_capenable = T4_CAP_ENABLE;
1215         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1216             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1217
1218         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1219         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1220         ifp->if_hw_tsomaxsegsize = 65536;
1221
1222         /* Initialize ifmedia for this VI */
1223         ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1224             cxgbe_media_status);
1225         build_medialist(vi->pi, &vi->media);
1226
1227         vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1228             EVENTHANDLER_PRI_ANY);
1229
1230         ether_ifattach(ifp, vi->hw_addr);
1231 #ifdef DEV_NETMAP
1232         if (vi->nnmrxq != 0)
1233                 cxgbe_nm_attach(vi);
1234 #endif
1235         sb = sbuf_new_auto();
1236         sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1237 #ifdef TCP_OFFLOAD
1238         if (ifp->if_capabilities & IFCAP_TOE)
1239                 sbuf_printf(sb, "; %d txq, %d rxq (TOE)",
1240                     vi->nofldtxq, vi->nofldrxq);
1241 #endif
1242 #ifdef DEV_NETMAP
1243         if (ifp->if_capabilities & IFCAP_NETMAP)
1244                 sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
1245                     vi->nnmtxq, vi->nnmrxq);
1246 #endif
1247         sbuf_finish(sb);
1248         device_printf(dev, "%s\n", sbuf_data(sb));
1249         sbuf_delete(sb);
1250
1251         vi_sysctls(vi);
1252
1253         return (0);
1254 }
1255
1256 static int
1257 cxgbe_attach(device_t dev)
1258 {
1259         struct port_info *pi = device_get_softc(dev);
1260         struct vi_info *vi;
1261         int i, rc;
1262
1263         callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1264
1265         rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1266         if (rc)
1267                 return (rc);
1268
1269         for_each_vi(pi, i, vi) {
1270                 if (i == 0)
1271                         continue;
1272                 vi->dev = device_add_child(dev, is_t4(pi->adapter) ?
1273                     "vcxgbe" : "vcxl", -1);
1274                 if (vi->dev == NULL) {
1275                         device_printf(dev, "failed to add VI %d\n", i);
1276                         continue;
1277                 }
1278                 device_set_softc(vi->dev, vi);
1279         }
1280
1281         cxgbe_sysctls(pi);
1282
1283         bus_generic_attach(dev);
1284
1285         return (0);
1286 }
1287
1288 static void
1289 cxgbe_vi_detach(struct vi_info *vi)
1290 {
1291         struct ifnet *ifp = vi->ifp;
1292
1293         ether_ifdetach(ifp);
1294
1295         if (vi->vlan_c)
1296                 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c);
1297
1298         /* Let detach proceed even if these fail. */
1299 #ifdef DEV_NETMAP
1300         if (ifp->if_capabilities & IFCAP_NETMAP)
1301                 cxgbe_nm_detach(vi);
1302 #endif
1303         cxgbe_uninit_synchronized(vi);
1304         callout_drain(&vi->tick);
1305         vi_full_uninit(vi);
1306
1307         ifmedia_removeall(&vi->media);
1308         if_free(vi->ifp);
1309         vi->ifp = NULL;
1310 }
1311
1312 static int
1313 cxgbe_detach(device_t dev)
1314 {
1315         struct port_info *pi = device_get_softc(dev);
1316         struct adapter *sc = pi->adapter;
1317         int rc;
1318
1319         /* Detach the extra VIs first. */
1320         rc = bus_generic_detach(dev);
1321         if (rc)
1322                 return (rc);
1323         device_delete_children(dev);
1324
1325         doom_vi(sc, &pi->vi[0]);
1326
1327         if (pi->flags & HAS_TRACEQ) {
1328                 sc->traceq = -1;        /* cloner should not create ifnet */
1329                 t4_tracer_port_detach(sc);
1330         }
1331
1332         cxgbe_vi_detach(&pi->vi[0]);
1333         callout_drain(&pi->tick);
1334
1335         end_synchronized_op(sc, 0);
1336
1337         return (0);
1338 }
1339
1340 static void
1341 cxgbe_init(void *arg)
1342 {
1343         struct vi_info *vi = arg;
1344         struct adapter *sc = vi->pi->adapter;
1345
1346         if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1347                 return;
1348         cxgbe_init_synchronized(vi);
1349         end_synchronized_op(sc, 0);
1350 }
1351
1352 static int
1353 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1354 {
1355         int rc = 0, mtu, flags, can_sleep;
1356         struct vi_info *vi = ifp->if_softc;
1357         struct adapter *sc = vi->pi->adapter;
1358         struct ifreq *ifr = (struct ifreq *)data;
1359         uint32_t mask;
1360
1361         switch (cmd) {
1362         case SIOCSIFMTU:
1363                 mtu = ifr->ifr_mtu;
1364                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1365                         return (EINVAL);
1366
1367                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1368                 if (rc)
1369                         return (rc);
1370                 ifp->if_mtu = mtu;
1371                 if (vi->flags & VI_INIT_DONE) {
1372                         t4_update_fl_bufsize(ifp);
1373                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1374                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1375                 }
1376                 end_synchronized_op(sc, 0);
1377                 break;
1378
1379         case SIOCSIFFLAGS:
1380                 can_sleep = 0;
1381 redo_sifflags:
1382                 rc = begin_synchronized_op(sc, vi,
1383                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1384                 if (rc)
1385                         return (rc);
1386
1387                 if (ifp->if_flags & IFF_UP) {
1388                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1389                                 flags = vi->if_flags;
1390                                 if ((ifp->if_flags ^ flags) &
1391                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1392                                         if (can_sleep == 1) {
1393                                                 end_synchronized_op(sc, 0);
1394                                                 can_sleep = 0;
1395                                                 goto redo_sifflags;
1396                                         }
1397                                         rc = update_mac_settings(ifp,
1398                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1399                                 }
1400                         } else {
1401                                 if (can_sleep == 0) {
1402                                         end_synchronized_op(sc, LOCK_HELD);
1403                                         can_sleep = 1;
1404                                         goto redo_sifflags;
1405                                 }
1406                                 rc = cxgbe_init_synchronized(vi);
1407                         }
1408                         vi->if_flags = ifp->if_flags;
1409                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1410                         if (can_sleep == 0) {
1411                                 end_synchronized_op(sc, LOCK_HELD);
1412                                 can_sleep = 1;
1413                                 goto redo_sifflags;
1414                         }
1415                         rc = cxgbe_uninit_synchronized(vi);
1416                 }
1417                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1418                 break;
1419
1420         case SIOCADDMULTI:
1421         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1422                 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi");
1423                 if (rc)
1424                         return (rc);
1425                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1426                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1427                 end_synchronized_op(sc, LOCK_HELD);
1428                 break;
1429
1430         case SIOCSIFCAP:
1431                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1432                 if (rc)
1433                         return (rc);
1434
1435                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1436                 if (mask & IFCAP_TXCSUM) {
1437                         ifp->if_capenable ^= IFCAP_TXCSUM;
1438                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1439
1440                         if (IFCAP_TSO4 & ifp->if_capenable &&
1441                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1442                                 ifp->if_capenable &= ~IFCAP_TSO4;
1443                                 if_printf(ifp,
1444                                     "tso4 disabled due to -txcsum.\n");
1445                         }
1446                 }
1447                 if (mask & IFCAP_TXCSUM_IPV6) {
1448                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1449                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1450
1451                         if (IFCAP_TSO6 & ifp->if_capenable &&
1452                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1453                                 ifp->if_capenable &= ~IFCAP_TSO6;
1454                                 if_printf(ifp,
1455                                     "tso6 disabled due to -txcsum6.\n");
1456                         }
1457                 }
1458                 if (mask & IFCAP_RXCSUM)
1459                         ifp->if_capenable ^= IFCAP_RXCSUM;
1460                 if (mask & IFCAP_RXCSUM_IPV6)
1461                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1462
1463                 /*
1464                  * Note that we leave CSUM_TSO alone (it is always set).  The
1465                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1466                  * sending a TSO request our way, so it's sufficient to toggle
1467                  * IFCAP_TSOx only.
1468                  */
1469                 if (mask & IFCAP_TSO4) {
1470                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1471                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1472                                 if_printf(ifp, "enable txcsum first.\n");
1473                                 rc = EAGAIN;
1474                                 goto fail;
1475                         }
1476                         ifp->if_capenable ^= IFCAP_TSO4;
1477                 }
1478                 if (mask & IFCAP_TSO6) {
1479                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1480                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1481                                 if_printf(ifp, "enable txcsum6 first.\n");
1482                                 rc = EAGAIN;
1483                                 goto fail;
1484                         }
1485                         ifp->if_capenable ^= IFCAP_TSO6;
1486                 }
1487                 if (mask & IFCAP_LRO) {
1488 #if defined(INET) || defined(INET6)
1489                         int i;
1490                         struct sge_rxq *rxq;
1491
1492                         ifp->if_capenable ^= IFCAP_LRO;
1493                         for_each_rxq(vi, i, rxq) {
1494                                 if (ifp->if_capenable & IFCAP_LRO)
1495                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1496                                 else
1497                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1498                         }
1499 #endif
1500                 }
1501 #ifdef TCP_OFFLOAD
1502                 if (mask & IFCAP_TOE) {
1503                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1504
1505                         rc = toe_capability(vi, enable);
1506                         if (rc != 0)
1507                                 goto fail;
1508
1509                         ifp->if_capenable ^= mask;
1510                 }
1511 #endif
1512                 if (mask & IFCAP_VLAN_HWTAGGING) {
1513                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1514                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1515                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1516                 }
1517                 if (mask & IFCAP_VLAN_MTU) {
1518                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1519
1520                         /* Need to find out how to disable auto-mtu-inflation */
1521                 }
1522                 if (mask & IFCAP_VLAN_HWTSO)
1523                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1524                 if (mask & IFCAP_VLAN_HWCSUM)
1525                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1526
1527 #ifdef VLAN_CAPABILITIES
1528                 VLAN_CAPABILITIES(ifp);
1529 #endif
1530 fail:
1531                 end_synchronized_op(sc, 0);
1532                 break;
1533
1534         case SIOCSIFMEDIA:
1535         case SIOCGIFMEDIA:
1536                 ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
1537                 break;
1538
1539         case SIOCGI2C: {
1540                 struct ifi2creq i2c;
1541
1542                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1543                 if (rc != 0)
1544                         break;
1545                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1546                         rc = EPERM;
1547                         break;
1548                 }
1549                 if (i2c.len > sizeof(i2c.data)) {
1550                         rc = EINVAL;
1551                         break;
1552                 }
1553                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1554                 if (rc)
1555                         return (rc);
1556                 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr,
1557                     i2c.offset, i2c.len, &i2c.data[0]);
1558                 end_synchronized_op(sc, 0);
1559                 if (rc == 0)
1560                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1561                 break;
1562         }
1563
1564         default:
1565                 rc = ether_ioctl(ifp, cmd, data);
1566         }
1567
1568         return (rc);
1569 }
1570
1571 static int
1572 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1573 {
1574         struct vi_info *vi = ifp->if_softc;
1575         struct port_info *pi = vi->pi;
1576         struct adapter *sc = pi->adapter;
1577         struct sge_txq *txq;
1578         void *items[1];
1579         int rc;
1580
1581         M_ASSERTPKTHDR(m);
1582         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1583
1584         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1585                 m_freem(m);
1586                 return (ENETDOWN);
1587         }
1588
1589         rc = parse_pkt(&m);
1590         if (__predict_false(rc != 0)) {
1591                 MPASS(m == NULL);                       /* was freed already */
1592                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1593                 return (rc);
1594         }
1595
1596         /* Select a txq. */
1597         txq = &sc->sge.txq[vi->first_txq];
1598         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1599                 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1600                     vi->rsrv_noflowq);
1601
1602         items[0] = m;
1603         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1604         if (__predict_false(rc != 0))
1605                 m_freem(m);
1606
1607         return (rc);
1608 }
1609
1610 static void
1611 cxgbe_qflush(struct ifnet *ifp)
1612 {
1613         struct vi_info *vi = ifp->if_softc;
1614         struct sge_txq *txq;
1615         int i;
1616
1617         /* queues do not exist if !VI_INIT_DONE. */
1618         if (vi->flags & VI_INIT_DONE) {
1619                 for_each_txq(vi, i, txq) {
1620                         TXQ_LOCK(txq);
1621                         txq->eq.flags &= ~EQ_ENABLED;
1622                         TXQ_UNLOCK(txq);
1623                         while (!mp_ring_is_idle(txq->r)) {
1624                                 mp_ring_check_drainage(txq->r, 0);
1625                                 pause("qflush", 1);
1626                         }
1627                 }
1628         }
1629         if_qflush(ifp);
1630 }
1631
1632 static int
1633 cxgbe_media_change(struct ifnet *ifp)
1634 {
1635         struct vi_info *vi = ifp->if_softc;
1636
1637         device_printf(vi->dev, "%s unimplemented.\n", __func__);
1638
1639         return (EOPNOTSUPP);
1640 }
1641
1642 static void
1643 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1644 {
1645         struct vi_info *vi = ifp->if_softc;
1646         struct port_info *pi = vi->pi;
1647         struct ifmedia_entry *cur;
1648         int speed = pi->link_cfg.speed;
1649
1650         cur = vi->media.ifm_cur;
1651
1652         ifmr->ifm_status = IFM_AVALID;
1653         if (!pi->link_cfg.link_ok)
1654                 return;
1655
1656         ifmr->ifm_status |= IFM_ACTIVE;
1657
1658         /* active and current will differ iff current media is autoselect. */
1659         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1660                 return;
1661
1662         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1663         if (speed == SPEED_10000)
1664                 ifmr->ifm_active |= IFM_10G_T;
1665         else if (speed == SPEED_1000)
1666                 ifmr->ifm_active |= IFM_1000_T;
1667         else if (speed == SPEED_100)
1668                 ifmr->ifm_active |= IFM_100_TX;
1669         else if (speed == SPEED_10)
1670                 ifmr->ifm_active |= IFM_10_T;
1671         else
1672                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1673                             speed));
1674 }
1675
1676 static int
1677 vcxgbe_probe(device_t dev)
1678 {
1679         char buf[128];
1680         struct vi_info *vi = device_get_softc(dev);
1681
1682         snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
1683             vi - vi->pi->vi);
1684         device_set_desc_copy(dev, buf);
1685
1686         return (BUS_PROBE_DEFAULT);
1687 }
1688
1689 static int
1690 vcxgbe_attach(device_t dev)
1691 {
1692         struct vi_info *vi;
1693         struct port_info *pi;
1694         struct adapter *sc;
1695         int func, index, rc;
1696         u32 param, val;
1697
1698         vi = device_get_softc(dev);
1699         pi = vi->pi;
1700         sc = pi->adapter;
1701
1702         index = vi - pi->vi;
1703         KASSERT(index < nitems(vi_mac_funcs),
1704             ("%s: VI %s doesn't have a MAC func", __func__,
1705             device_get_nameunit(dev)));
1706         func = vi_mac_funcs[index];
1707         rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
1708             vi->hw_addr, &vi->rss_size, func, 0);
1709         if (rc < 0) {
1710                 device_printf(dev, "Failed to allocate virtual interface "
1711                     "for port %d: %d\n", pi->port_id, -rc);
1712                 return (-rc);
1713         }
1714         vi->viid = rc;
1715
1716         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
1717             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
1718             V_FW_PARAMS_PARAM_YZ(vi->viid);
1719         rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1720         if (rc)
1721                 vi->rss_base = 0xffff;
1722         else {
1723                 /* MPASS((val >> 16) == rss_size); */
1724                 vi->rss_base = val & 0xffff;
1725         }
1726
1727         rc = cxgbe_vi_attach(dev, vi);
1728         if (rc) {
1729                 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1730                 return (rc);
1731         }
1732         return (0);
1733 }
1734
1735 static int
1736 vcxgbe_detach(device_t dev)
1737 {
1738         struct vi_info *vi;
1739         struct adapter *sc;
1740
1741         vi = device_get_softc(dev);
1742         sc = vi->pi->adapter;
1743
1744         doom_vi(sc, vi);
1745
1746         cxgbe_vi_detach(vi);
1747         t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1748
1749         end_synchronized_op(sc, 0);
1750
1751         return (0);
1752 }
1753
1754 void
1755 t4_fatal_err(struct adapter *sc)
1756 {
1757         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1758         t4_intr_disable(sc);
1759         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1760             device_get_nameunit(sc->dev));
1761 }
1762
1763 static int
1764 map_bars_0_and_4(struct adapter *sc)
1765 {
1766         sc->regs_rid = PCIR_BAR(0);
1767         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1768             &sc->regs_rid, RF_ACTIVE);
1769         if (sc->regs_res == NULL) {
1770                 device_printf(sc->dev, "cannot map registers.\n");
1771                 return (ENXIO);
1772         }
1773         sc->bt = rman_get_bustag(sc->regs_res);
1774         sc->bh = rman_get_bushandle(sc->regs_res);
1775         sc->mmio_len = rman_get_size(sc->regs_res);
1776         setbit(&sc->doorbells, DOORBELL_KDB);
1777
1778         sc->msix_rid = PCIR_BAR(4);
1779         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1780             &sc->msix_rid, RF_ACTIVE);
1781         if (sc->msix_res == NULL) {
1782                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1783                 return (ENXIO);
1784         }
1785
1786         return (0);
1787 }
1788
1789 static int
1790 map_bar_2(struct adapter *sc)
1791 {
1792
1793         /*
1794          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1795          * to map it if RDMA is disabled.
1796          */
1797         if (is_t4(sc) && sc->rdmacaps == 0)
1798                 return (0);
1799
1800         sc->udbs_rid = PCIR_BAR(2);
1801         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1802             &sc->udbs_rid, RF_ACTIVE);
1803         if (sc->udbs_res == NULL) {
1804                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1805                 return (ENXIO);
1806         }
1807         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1808
1809         if (is_t5(sc)) {
1810                 setbit(&sc->doorbells, DOORBELL_UDB);
1811 #if defined(__i386__) || defined(__amd64__)
1812                 if (t5_write_combine) {
1813                         int rc;
1814
1815                         /*
1816                          * Enable write combining on BAR2.  This is the
1817                          * userspace doorbell BAR and is split into 128B
1818                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1819                          * with an egress queue.  The first 64B has the doorbell
1820                          * and the second 64B can be used to submit a tx work
1821                          * request with an implicit doorbell.
1822                          */
1823
1824                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1825                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1826                         if (rc == 0) {
1827                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1828                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1829                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1830                         } else {
1831                                 device_printf(sc->dev,
1832                                     "couldn't enable write combining: %d\n",
1833                                     rc);
1834                         }
1835
1836                         t4_write_reg(sc, A_SGE_STAT_CFG,
1837                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1838                 }
1839 #endif
1840         }
1841
1842         return (0);
1843 }
1844
1845 static const struct memwin t4_memwin[] = {
1846         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1847         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1848         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1849 };
1850
1851 static const struct memwin t5_memwin[] = {
1852         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1853         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1854         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1855 };
1856
1857 static void
1858 setup_memwin(struct adapter *sc)
1859 {
1860         const struct memwin *mw;
1861         int i, n;
1862         uint32_t bar0;
1863
1864         if (is_t4(sc)) {
1865                 /*
1866                  * Read low 32b of bar0 indirectly via the hardware backdoor
1867                  * mechanism.  Works from within PCI passthrough environments
1868                  * too, where rman_get_start() can return a different value.  We
1869                  * need to program the T4 memory window decoders with the actual
1870                  * addresses that will be coming across the PCIe link.
1871                  */
1872                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1873                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1874
1875                 mw = &t4_memwin[0];
1876                 n = nitems(t4_memwin);
1877         } else {
1878                 /* T5 uses the relative offset inside the PCIe BAR */
1879                 bar0 = 0;
1880
1881                 mw = &t5_memwin[0];
1882                 n = nitems(t5_memwin);
1883         }
1884
1885         for (i = 0; i < n; i++, mw++) {
1886                 t4_write_reg(sc,
1887                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1888                     (mw->base + bar0) | V_BIR(0) |
1889                     V_WINDOW(ilog2(mw->aperture) - 10));
1890         }
1891
1892         /* flush */
1893         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1894 }
1895
1896 /*
1897  * Verify that the memory range specified by the addr/len pair is valid and lies
1898  * entirely within a single region (EDCx or MCx).
1899  */
1900 static int
1901 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1902 {
1903         uint32_t em, addr_len, maddr, mlen;
1904
1905         /* Memory can only be accessed in naturally aligned 4 byte units */
1906         if (addr & 3 || len & 3 || len == 0)
1907                 return (EINVAL);
1908
1909         /* Enabled memories */
1910         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1911         if (em & F_EDRAM0_ENABLE) {
1912                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1913                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1914                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1915                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1916                     addr + len <= maddr + mlen)
1917                         return (0);
1918         }
1919         if (em & F_EDRAM1_ENABLE) {
1920                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1921                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1922                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1923                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1924                     addr + len <= maddr + mlen)
1925                         return (0);
1926         }
1927         if (em & F_EXT_MEM_ENABLE) {
1928                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1929                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1930                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1931                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1932                     addr + len <= maddr + mlen)
1933                         return (0);
1934         }
1935         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1936                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1937                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1938                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1939                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1940                     addr + len <= maddr + mlen)
1941                         return (0);
1942         }
1943
1944         return (EFAULT);
1945 }
1946
1947 static int
1948 fwmtype_to_hwmtype(int mtype)
1949 {
1950
1951         switch (mtype) {
1952         case FW_MEMTYPE_EDC0:
1953                 return (MEM_EDC0);
1954         case FW_MEMTYPE_EDC1:
1955                 return (MEM_EDC1);
1956         case FW_MEMTYPE_EXTMEM:
1957                 return (MEM_MC0);
1958         case FW_MEMTYPE_EXTMEM1:
1959                 return (MEM_MC1);
1960         default:
1961                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1962         }
1963 }
1964
1965 /*
1966  * Verify that the memory range specified by the memtype/offset/len pair is
1967  * valid and lies entirely within the memtype specified.  The global address of
1968  * the start of the range is returned in addr.
1969  */
1970 static int
1971 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1972     uint32_t *addr)
1973 {
1974         uint32_t em, addr_len, maddr, mlen;
1975
1976         /* Memory can only be accessed in naturally aligned 4 byte units */
1977         if (off & 3 || len & 3 || len == 0)
1978                 return (EINVAL);
1979
1980         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1981         switch (fwmtype_to_hwmtype(mtype)) {
1982         case MEM_EDC0:
1983                 if (!(em & F_EDRAM0_ENABLE))
1984                         return (EINVAL);
1985                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1986                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1987                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1988                 break;
1989         case MEM_EDC1:
1990                 if (!(em & F_EDRAM1_ENABLE))
1991                         return (EINVAL);
1992                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1993                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1994                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1995                 break;
1996         case MEM_MC:
1997                 if (!(em & F_EXT_MEM_ENABLE))
1998                         return (EINVAL);
1999                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2000                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
2001                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
2002                 break;
2003         case MEM_MC1:
2004                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
2005                         return (EINVAL);
2006                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2007                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2008                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
2009                 break;
2010         default:
2011                 return (EINVAL);
2012         }
2013
2014         if (mlen > 0 && off < mlen && off + len <= mlen) {
2015                 *addr = maddr + off;    /* global address */
2016                 return (0);
2017         }
2018
2019         return (EFAULT);
2020 }
2021
2022 static void
2023 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
2024 {
2025         const struct memwin *mw;
2026
2027         if (is_t4(sc)) {
2028                 KASSERT(win >= 0 && win < nitems(t4_memwin),
2029                     ("%s: incorrect memwin# (%d)", __func__, win));
2030                 mw = &t4_memwin[win];
2031         } else {
2032                 KASSERT(win >= 0 && win < nitems(t5_memwin),
2033                     ("%s: incorrect memwin# (%d)", __func__, win));
2034                 mw = &t5_memwin[win];
2035         }
2036
2037         if (base != NULL)
2038                 *base = mw->base;
2039         if (aperture != NULL)
2040                 *aperture = mw->aperture;
2041 }
2042
2043 /*
2044  * Positions the memory window such that it can be used to access the specified
2045  * address in the chip's address space.  The return value is the offset of addr
2046  * from the start of the window.
2047  */
2048 static uint32_t
2049 position_memwin(struct adapter *sc, int n, uint32_t addr)
2050 {
2051         uint32_t start, pf;
2052         uint32_t reg;
2053
2054         KASSERT(n >= 0 && n <= 3,
2055             ("%s: invalid window %d.", __func__, n));
2056         KASSERT((addr & 3) == 0,
2057             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
2058
2059         if (is_t4(sc)) {
2060                 pf = 0;
2061                 start = addr & ~0xf;    /* start must be 16B aligned */
2062         } else {
2063                 pf = V_PFNUM(sc->pf);
2064                 start = addr & ~0x7f;   /* start must be 128B aligned */
2065         }
2066         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
2067
2068         t4_write_reg(sc, reg, start | pf);
2069         t4_read_reg(sc, reg);
2070
2071         return (addr - start);
2072 }
2073
2074 static int
2075 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis,
2076     struct intrs_and_queues *iaq)
2077 {
2078         int rc, itype, navail, nrxq10g, nrxq1g, n;
2079         int nofldrxq10g = 0, nofldrxq1g = 0;
2080
2081         bzero(iaq, sizeof(*iaq));
2082
2083         iaq->ntxq10g = t4_ntxq10g;
2084         iaq->ntxq1g = t4_ntxq1g;
2085         iaq->ntxq_vi = t4_ntxq_vi;
2086         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
2087         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
2088         iaq->nrxq_vi = t4_nrxq_vi;
2089         iaq->rsrv_noflowq = t4_rsrv_noflowq;
2090 #ifdef TCP_OFFLOAD
2091         if (is_offload(sc)) {
2092                 iaq->nofldtxq10g = t4_nofldtxq10g;
2093                 iaq->nofldtxq1g = t4_nofldtxq1g;
2094                 iaq->nofldtxq_vi = t4_nofldtxq_vi;
2095                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
2096                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
2097                 iaq->nofldrxq_vi = t4_nofldrxq_vi;
2098         }
2099 #endif
2100 #ifdef DEV_NETMAP
2101         iaq->nnmtxq_vi = t4_nnmtxq_vi;
2102         iaq->nnmrxq_vi = t4_nnmrxq_vi;
2103 #endif
2104
2105         for (itype = INTR_MSIX; itype; itype >>= 1) {
2106
2107                 if ((itype & t4_intr_types) == 0)
2108                         continue;       /* not allowed */
2109
2110                 if (itype == INTR_MSIX)
2111                         navail = pci_msix_count(sc->dev);
2112                 else if (itype == INTR_MSI)
2113                         navail = pci_msi_count(sc->dev);
2114                 else
2115                         navail = 1;
2116 restart:
2117                 if (navail == 0)
2118                         continue;
2119
2120                 iaq->intr_type = itype;
2121                 iaq->intr_flags_10g = 0;
2122                 iaq->intr_flags_1g = 0;
2123
2124                 /*
2125                  * Best option: an interrupt vector for errors, one for the
2126                  * firmware event queue, and one for every rxq (NIC and TOE) of
2127                  * every VI.  The VIs that support netmap use the same
2128                  * interrupts for the NIC rx queues and the netmap rx queues
2129                  * because only one set of queues is active at a time.
2130                  */
2131                 iaq->nirq = T4_EXTRA_INTR;
2132                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2133                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2134                 iaq->nirq += (n10g + n1g) * (num_vis - 1) *
2135                     max(iaq->nrxq_vi, iaq->nnmrxq_vi);  /* See comment above. */
2136                 iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi;
2137                 if (iaq->nirq <= navail &&
2138                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2139                         iaq->intr_flags_10g = INTR_ALL;
2140                         iaq->intr_flags_1g = INTR_ALL;
2141                         goto allocate;
2142                 }
2143
2144                 /* Disable the VIs (and netmap) if there aren't enough intrs */
2145                 if (num_vis > 1) {
2146                         device_printf(sc->dev, "virtual interfaces disabled "
2147                             "because num_vis=%u with current settings "
2148                             "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, "
2149                             "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, "
2150                             "nnmrxq_vi=%u) would need %u interrupts but "
2151                             "only %u are available.\n", num_vis, nrxq10g,
2152                             nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi,
2153                             iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq,
2154                             navail);
2155                         num_vis = 1;
2156                         iaq->ntxq_vi = iaq->nrxq_vi = 0;
2157                         iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
2158                         iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
2159                         goto restart;
2160                 }
2161
2162                 /*
2163                  * Second best option: a vector for errors, one for the firmware
2164                  * event queue, and vectors for either all the NIC rx queues or
2165                  * all the TOE rx queues.  The queues that don't get vectors
2166                  * will forward their interrupts to those that do.
2167                  */
2168                 iaq->nirq = T4_EXTRA_INTR;
2169                 if (nrxq10g >= nofldrxq10g) {
2170                         iaq->intr_flags_10g = INTR_RXQ;
2171                         iaq->nirq += n10g * nrxq10g;
2172                 } else {
2173                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2174                         iaq->nirq += n10g * nofldrxq10g;
2175                 }
2176                 if (nrxq1g >= nofldrxq1g) {
2177                         iaq->intr_flags_1g = INTR_RXQ;
2178                         iaq->nirq += n1g * nrxq1g;
2179                 } else {
2180                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2181                         iaq->nirq += n1g * nofldrxq1g;
2182                 }
2183                 if (iaq->nirq <= navail &&
2184                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2185                         goto allocate;
2186
2187                 /*
2188                  * Next best option: an interrupt vector for errors, one for the
2189                  * firmware event queue, and at least one per main-VI.  At this
2190                  * point we know we'll have to downsize nrxq and/or nofldrxq to
2191                  * fit what's available to us.
2192                  */
2193                 iaq->nirq = T4_EXTRA_INTR;
2194                 iaq->nirq += n10g + n1g;
2195                 if (iaq->nirq <= navail) {
2196                         int leftover = navail - iaq->nirq;
2197
2198                         if (n10g > 0) {
2199                                 int target = max(nrxq10g, nofldrxq10g);
2200
2201                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2202                                     INTR_RXQ : INTR_OFLD_RXQ;
2203
2204                                 n = 1;
2205                                 while (n < target && leftover >= n10g) {
2206                                         leftover -= n10g;
2207                                         iaq->nirq += n10g;
2208                                         n++;
2209                                 }
2210                                 iaq->nrxq10g = min(n, nrxq10g);
2211 #ifdef TCP_OFFLOAD
2212                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2213 #endif
2214                         }
2215
2216                         if (n1g > 0) {
2217                                 int target = max(nrxq1g, nofldrxq1g);
2218
2219                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2220                                     INTR_RXQ : INTR_OFLD_RXQ;
2221
2222                                 n = 1;
2223                                 while (n < target && leftover >= n1g) {
2224                                         leftover -= n1g;
2225                                         iaq->nirq += n1g;
2226                                         n++;
2227                                 }
2228                                 iaq->nrxq1g = min(n, nrxq1g);
2229 #ifdef TCP_OFFLOAD
2230                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2231 #endif
2232                         }
2233
2234                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2235                                 goto allocate;
2236                 }
2237
2238                 /*
2239                  * Least desirable option: one interrupt vector for everything.
2240                  */
2241                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2242                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2243 #ifdef TCP_OFFLOAD
2244                 if (is_offload(sc))
2245                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2246 #endif
2247 allocate:
2248                 navail = iaq->nirq;
2249                 rc = 0;
2250                 if (itype == INTR_MSIX)
2251                         rc = pci_alloc_msix(sc->dev, &navail);
2252                 else if (itype == INTR_MSI)
2253                         rc = pci_alloc_msi(sc->dev, &navail);
2254
2255                 if (rc == 0) {
2256                         if (navail == iaq->nirq)
2257                                 return (0);
2258
2259                         /*
2260                          * Didn't get the number requested.  Use whatever number
2261                          * the kernel is willing to allocate (it's in navail).
2262                          */
2263                         device_printf(sc->dev, "fewer vectors than requested, "
2264                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2265                             itype, iaq->nirq, navail);
2266                         pci_release_msi(sc->dev);
2267                         goto restart;
2268                 }
2269
2270                 device_printf(sc->dev,
2271                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2272                     itype, rc, iaq->nirq, navail);
2273         }
2274
2275         device_printf(sc->dev,
2276             "failed to find a usable interrupt type.  "
2277             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2278             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2279
2280         return (ENXIO);
2281 }
2282
2283 #define FW_VERSION(chip) ( \
2284     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2285     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2286     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2287     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2288 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2289
2290 struct fw_info {
2291         uint8_t chip;
2292         char *kld_name;
2293         char *fw_mod_name;
2294         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2295 } fw_info[] = {
2296         {
2297                 .chip = CHELSIO_T4,
2298                 .kld_name = "t4fw_cfg",
2299                 .fw_mod_name = "t4fw",
2300                 .fw_hdr = {
2301                         .chip = FW_HDR_CHIP_T4,
2302                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2303                         .intfver_nic = FW_INTFVER(T4, NIC),
2304                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2305                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2306                         .intfver_ri = FW_INTFVER(T4, RI),
2307                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2308                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2309                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2310                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2311                 },
2312         }, {
2313                 .chip = CHELSIO_T5,
2314                 .kld_name = "t5fw_cfg",
2315                 .fw_mod_name = "t5fw",
2316                 .fw_hdr = {
2317                         .chip = FW_HDR_CHIP_T5,
2318                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2319                         .intfver_nic = FW_INTFVER(T5, NIC),
2320                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2321                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2322                         .intfver_ri = FW_INTFVER(T5, RI),
2323                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2324                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2325                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2326                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2327                 },
2328         }
2329 };
2330
2331 static struct fw_info *
2332 find_fw_info(int chip)
2333 {
2334         int i;
2335
2336         for (i = 0; i < nitems(fw_info); i++) {
2337                 if (fw_info[i].chip == chip)
2338                         return (&fw_info[i]);
2339         }
2340         return (NULL);
2341 }
2342
2343 /*
2344  * Is the given firmware API compatible with the one the driver was compiled
2345  * with?
2346  */
2347 static int
2348 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2349 {
2350
2351         /* short circuit if it's the exact same firmware version */
2352         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2353                 return (1);
2354
2355         /*
2356          * XXX: Is this too conservative?  Perhaps I should limit this to the
2357          * features that are supported in the driver.
2358          */
2359 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2360         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2361             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2362             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2363                 return (1);
2364 #undef SAME_INTF
2365
2366         return (0);
2367 }
2368
2369 /*
2370  * The firmware in the KLD is usable, but should it be installed?  This routine
2371  * explains itself in detail if it indicates the KLD firmware should be
2372  * installed.
2373  */
2374 static int
2375 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2376 {
2377         const char *reason;
2378
2379         if (!card_fw_usable) {
2380                 reason = "incompatible or unusable";
2381                 goto install;
2382         }
2383
2384         if (k > c) {
2385                 reason = "older than the version bundled with this driver";
2386                 goto install;
2387         }
2388
2389         if (t4_fw_install == 2 && k != c) {
2390                 reason = "different than the version bundled with this driver";
2391                 goto install;
2392         }
2393
2394         return (0);
2395
2396 install:
2397         if (t4_fw_install == 0) {
2398                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2399                     "but the driver is prohibited from installing a different "
2400                     "firmware on the card.\n",
2401                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2402                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2403
2404                 return (0);
2405         }
2406
2407         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2408             "installing firmware %u.%u.%u.%u on card.\n",
2409             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2410             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2411             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2412             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2413
2414         return (1);
2415 }
2416 /*
2417  * Establish contact with the firmware and determine if we are the master driver
2418  * or not, and whether we are responsible for chip initialization.
2419  */
2420 static int
2421 prep_firmware(struct adapter *sc)
2422 {
2423         const struct firmware *fw = NULL, *default_cfg;
2424         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2425         enum dev_state state;
2426         struct fw_info *fw_info;
2427         struct fw_hdr *card_fw;         /* fw on the card */
2428         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2429         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2430                                            against */
2431
2432         /* Contact firmware. */
2433         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2434         if (rc < 0 || state == DEV_STATE_ERR) {
2435                 rc = -rc;
2436                 device_printf(sc->dev,
2437                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2438                 return (rc);
2439         }
2440         pf = rc;
2441         if (pf == sc->mbox)
2442                 sc->flags |= MASTER_PF;
2443         else if (state == DEV_STATE_UNINIT) {
2444                 /*
2445                  * We didn't get to be the master so we definitely won't be
2446                  * configuring the chip.  It's a bug if someone else hasn't
2447                  * configured it already.
2448                  */
2449                 device_printf(sc->dev, "couldn't be master(%d), "
2450                     "device not already initialized either(%d).\n", rc, state);
2451                 return (EDOOFUS);
2452         }
2453
2454         /* This is the firmware whose headers the driver was compiled against */
2455         fw_info = find_fw_info(chip_id(sc));
2456         if (fw_info == NULL) {
2457                 device_printf(sc->dev,
2458                     "unable to look up firmware information for chip %d.\n",
2459                     chip_id(sc));
2460                 return (EINVAL);
2461         }
2462         drv_fw = &fw_info->fw_hdr;
2463
2464         /*
2465          * The firmware KLD contains many modules.  The KLD name is also the
2466          * name of the module that contains the default config file.
2467          */
2468         default_cfg = firmware_get(fw_info->kld_name);
2469
2470         /* Read the header of the firmware on the card */
2471         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2472         rc = -t4_read_flash(sc, FLASH_FW_START,
2473             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2474         if (rc == 0)
2475                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2476         else {
2477                 device_printf(sc->dev,
2478                     "Unable to read card's firmware header: %d\n", rc);
2479                 card_fw_usable = 0;
2480         }
2481
2482         /* This is the firmware in the KLD */
2483         fw = firmware_get(fw_info->fw_mod_name);
2484         if (fw != NULL) {
2485                 kld_fw = (const void *)fw->data;
2486                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2487         } else {
2488                 kld_fw = NULL;
2489                 kld_fw_usable = 0;
2490         }
2491
2492         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2493             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2494                 /*
2495                  * Common case: the firmware on the card is an exact match and
2496                  * the KLD is an exact match too, or the KLD is
2497                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2498                  * here -- use cxgbetool loadfw if you want to reinstall the
2499                  * same firmware as the one on the card.
2500                  */
2501         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2502             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2503             be32toh(card_fw->fw_ver))) {
2504
2505                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2506                 if (rc != 0) {
2507                         device_printf(sc->dev,
2508                             "failed to install firmware: %d\n", rc);
2509                         goto done;
2510                 }
2511
2512                 /* Installed successfully, update the cached header too. */
2513                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2514                 card_fw_usable = 1;
2515                 need_fw_reset = 0;      /* already reset as part of load_fw */
2516         }
2517
2518         if (!card_fw_usable) {
2519                 uint32_t d, c, k;
2520
2521                 d = ntohl(drv_fw->fw_ver);
2522                 c = ntohl(card_fw->fw_ver);
2523                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2524
2525                 device_printf(sc->dev, "Cannot find a usable firmware: "
2526                     "fw_install %d, chip state %d, "
2527                     "driver compiled with %d.%d.%d.%d, "
2528                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2529                     t4_fw_install, state,
2530                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2531                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2532                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2533                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2534                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2535                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2536                 rc = EINVAL;
2537                 goto done;
2538         }
2539
2540         /* We're using whatever's on the card and it's known to be good. */
2541         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2542         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2543             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2544             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2545             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2546             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2547         t4_get_tp_version(sc, &sc->params.tp_vers);
2548
2549         /* Reset device */
2550         if (need_fw_reset &&
2551             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2552                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2553                 if (rc != ETIMEDOUT && rc != EIO)
2554                         t4_fw_bye(sc, sc->mbox);
2555                 goto done;
2556         }
2557         sc->flags |= FW_OK;
2558
2559         rc = get_params__pre_init(sc);
2560         if (rc != 0)
2561                 goto done; /* error message displayed already */
2562
2563         /* Partition adapter resources as specified in the config file. */
2564         if (state == DEV_STATE_UNINIT) {
2565
2566                 KASSERT(sc->flags & MASTER_PF,
2567                     ("%s: trying to change chip settings when not master.",
2568                     __func__));
2569
2570                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2571                 if (rc != 0)
2572                         goto done;      /* error message displayed already */
2573
2574                 t4_tweak_chip_settings(sc);
2575
2576                 /* get basic stuff going */
2577                 rc = -t4_fw_initialize(sc, sc->mbox);
2578                 if (rc != 0) {
2579                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2580                         goto done;
2581                 }
2582         } else {
2583                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2584                 sc->cfcsum = 0;
2585         }
2586
2587 done:
2588         free(card_fw, M_CXGBE);
2589         if (fw != NULL)
2590                 firmware_put(fw, FIRMWARE_UNLOAD);
2591         if (default_cfg != NULL)
2592                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2593
2594         return (rc);
2595 }
2596
2597 #define FW_PARAM_DEV(param) \
2598         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2599          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2600 #define FW_PARAM_PFVF(param) \
2601         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2602          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2603
2604 /*
2605  * Partition chip resources for use between various PFs, VFs, etc.
2606  */
2607 static int
2608 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2609     const char *name_prefix)
2610 {
2611         const struct firmware *cfg = NULL;
2612         int rc = 0;
2613         struct fw_caps_config_cmd caps;
2614         uint32_t mtype, moff, finicsum, cfcsum;
2615
2616         /*
2617          * Figure out what configuration file to use.  Pick the default config
2618          * file for the card if the user hasn't specified one explicitly.
2619          */
2620         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2621         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2622                 /* Card specific overrides go here. */
2623                 if (pci_get_device(sc->dev) == 0x440a)
2624                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2625                 if (is_fpga(sc))
2626                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2627         }
2628
2629         /*
2630          * We need to load another module if the profile is anything except
2631          * "default" or "flash".
2632          */
2633         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2634             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2635                 char s[32];
2636
2637                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2638                 cfg = firmware_get(s);
2639                 if (cfg == NULL) {
2640                         if (default_cfg != NULL) {
2641                                 device_printf(sc->dev,
2642                                     "unable to load module \"%s\" for "
2643                                     "configuration profile \"%s\", will use "
2644                                     "the default config file instead.\n",
2645                                     s, sc->cfg_file);
2646                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2647                                     "%s", DEFAULT_CF);
2648                         } else {
2649                                 device_printf(sc->dev,
2650                                     "unable to load module \"%s\" for "
2651                                     "configuration profile \"%s\", will use "
2652                                     "the config file on the card's flash "
2653                                     "instead.\n", s, sc->cfg_file);
2654                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2655                                     "%s", FLASH_CF);
2656                         }
2657                 }
2658         }
2659
2660         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2661             default_cfg == NULL) {
2662                 device_printf(sc->dev,
2663                     "default config file not available, will use the config "
2664                     "file on the card's flash instead.\n");
2665                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2666         }
2667
2668         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2669                 u_int cflen, i, n;
2670                 const uint32_t *cfdata;
2671                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2672
2673                 KASSERT(cfg != NULL || default_cfg != NULL,
2674                     ("%s: no config to upload", __func__));
2675
2676                 /*
2677                  * Ask the firmware where it wants us to upload the config file.
2678                  */
2679                 param = FW_PARAM_DEV(CF);
2680                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2681                 if (rc != 0) {
2682                         /* No support for config file?  Shouldn't happen. */
2683                         device_printf(sc->dev,
2684                             "failed to query config file location: %d.\n", rc);
2685                         goto done;
2686                 }
2687                 mtype = G_FW_PARAMS_PARAM_Y(val);
2688                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2689
2690                 /*
2691                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2692                  * useless stuffing/comments at the end of the config file so
2693                  * it's ok to simply throw away the last remaining bytes when
2694                  * the config file is not an exact multiple of 4.  This also
2695                  * helps with the validate_mt_off_len check.
2696                  */
2697                 if (cfg != NULL) {
2698                         cflen = cfg->datasize & ~3;
2699                         cfdata = cfg->data;
2700                 } else {
2701                         cflen = default_cfg->datasize & ~3;
2702                         cfdata = default_cfg->data;
2703                 }
2704
2705                 if (cflen > FLASH_CFG_MAX_SIZE) {
2706                         device_printf(sc->dev,
2707                             "config file too long (%d, max allowed is %d).  "
2708                             "Will try to use the config on the card, if any.\n",
2709                             cflen, FLASH_CFG_MAX_SIZE);
2710                         goto use_config_on_flash;
2711                 }
2712
2713                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2714                 if (rc != 0) {
2715                         device_printf(sc->dev,
2716                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2717                             "Will try to use the config on the card, if any.\n",
2718                             __func__, mtype, moff, cflen, rc);
2719                         goto use_config_on_flash;
2720                 }
2721
2722                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2723                 while (cflen) {
2724                         off = position_memwin(sc, 2, addr);
2725                         n = min(cflen, mw_aperture - off);
2726                         for (i = 0; i < n; i += 4)
2727                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2728                         cflen -= n;
2729                         addr += n;
2730                 }
2731         } else {
2732 use_config_on_flash:
2733                 mtype = FW_MEMTYPE_FLASH;
2734                 moff = t4_flash_cfg_addr(sc);
2735         }
2736
2737         bzero(&caps, sizeof(caps));
2738         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2739             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2740         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2741             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2742             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2743         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2744         if (rc != 0) {
2745                 device_printf(sc->dev,
2746                     "failed to pre-process config file: %d "
2747                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2748                 goto done;
2749         }
2750
2751         finicsum = be32toh(caps.finicsum);
2752         cfcsum = be32toh(caps.cfcsum);
2753         if (finicsum != cfcsum) {
2754                 device_printf(sc->dev,
2755                     "WARNING: config file checksum mismatch: %08x %08x\n",
2756                     finicsum, cfcsum);
2757         }
2758         sc->cfcsum = cfcsum;
2759
2760 #define LIMIT_CAPS(x) do { \
2761         caps.x &= htobe16(t4_##x##_allowed); \
2762 } while (0)
2763
2764         /*
2765          * Let the firmware know what features will (not) be used so it can tune
2766          * things accordingly.
2767          */
2768         LIMIT_CAPS(linkcaps);
2769         LIMIT_CAPS(niccaps);
2770         LIMIT_CAPS(toecaps);
2771         LIMIT_CAPS(rdmacaps);
2772         LIMIT_CAPS(iscsicaps);
2773         LIMIT_CAPS(fcoecaps);
2774 #undef LIMIT_CAPS
2775
2776         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2777             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2778         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2779         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2780         if (rc != 0) {
2781                 device_printf(sc->dev,
2782                     "failed to process config file: %d.\n", rc);
2783         }
2784 done:
2785         if (cfg != NULL)
2786                 firmware_put(cfg, FIRMWARE_UNLOAD);
2787         return (rc);
2788 }
2789
2790 /*
2791  * Retrieve parameters that are needed (or nice to have) very early.
2792  */
2793 static int
2794 get_params__pre_init(struct adapter *sc)
2795 {
2796         int rc;
2797         uint32_t param[2], val[2];
2798         struct fw_devlog_cmd cmd;
2799         struct devlog_params *dlog = &sc->params.devlog;
2800
2801         param[0] = FW_PARAM_DEV(PORTVEC);
2802         param[1] = FW_PARAM_DEV(CCLK);
2803         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2804         if (rc != 0) {
2805                 device_printf(sc->dev,
2806                     "failed to query parameters (pre_init): %d.\n", rc);
2807                 return (rc);
2808         }
2809
2810         sc->params.portvec = val[0];
2811         sc->params.nports = bitcount32(val[0]);
2812         sc->params.vpd.cclk = val[1];
2813
2814         /* Read device log parameters. */
2815         bzero(&cmd, sizeof(cmd));
2816         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2817             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2818         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2819         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2820         if (rc != 0) {
2821                 device_printf(sc->dev,
2822                     "failed to get devlog parameters: %d.\n", rc);
2823                 bzero(dlog, sizeof (*dlog));
2824                 rc = 0; /* devlog isn't critical for device operation */
2825         } else {
2826                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2827                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2828                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2829                 dlog->size = be32toh(cmd.memsize_devlog);
2830         }
2831
2832         return (rc);
2833 }
2834
2835 /*
2836  * Retrieve various parameters that are of interest to the driver.  The device
2837  * has been initialized by the firmware at this point.
2838  */
2839 static int
2840 get_params__post_init(struct adapter *sc)
2841 {
2842         int rc;
2843         uint32_t param[7], val[7];
2844         struct fw_caps_config_cmd caps;
2845
2846         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2847         param[1] = FW_PARAM_PFVF(EQ_START);
2848         param[2] = FW_PARAM_PFVF(FILTER_START);
2849         param[3] = FW_PARAM_PFVF(FILTER_END);
2850         param[4] = FW_PARAM_PFVF(L2T_START);
2851         param[5] = FW_PARAM_PFVF(L2T_END);
2852         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2853         if (rc != 0) {
2854                 device_printf(sc->dev,
2855                     "failed to query parameters (post_init): %d.\n", rc);
2856                 return (rc);
2857         }
2858
2859         sc->sge.iq_start = val[0];
2860         sc->sge.eq_start = val[1];
2861         sc->tids.ftid_base = val[2];
2862         sc->tids.nftids = val[3] - val[2] + 1;
2863         sc->params.ftid_min = val[2];
2864         sc->params.ftid_max = val[3];
2865         sc->vres.l2t.start = val[4];
2866         sc->vres.l2t.size = val[5] - val[4] + 1;
2867         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2868             ("%s: L2 table size (%u) larger than expected (%u)",
2869             __func__, sc->vres.l2t.size, L2T_SIZE));
2870
2871         /* get capabilites */
2872         bzero(&caps, sizeof(caps));
2873         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2874             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2875         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2876         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2877         if (rc != 0) {
2878                 device_printf(sc->dev,
2879                     "failed to get card capabilities: %d.\n", rc);
2880                 return (rc);
2881         }
2882
2883 #define READ_CAPS(x) do { \
2884         sc->x = htobe16(caps.x); \
2885 } while (0)
2886         READ_CAPS(linkcaps);
2887         READ_CAPS(niccaps);
2888         READ_CAPS(toecaps);
2889         READ_CAPS(rdmacaps);
2890         READ_CAPS(iscsicaps);
2891         READ_CAPS(fcoecaps);
2892
2893         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2894                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2895                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2896                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2897                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2898                 if (rc != 0) {
2899                         device_printf(sc->dev,
2900                             "failed to query NIC parameters: %d.\n", rc);
2901                         return (rc);
2902                 }
2903                 sc->tids.etid_base = val[0];
2904                 sc->params.etid_min = val[0];
2905                 sc->tids.netids = val[1] - val[0] + 1;
2906                 sc->params.netids = sc->tids.netids;
2907                 sc->params.eo_wr_cred = val[2];
2908                 sc->params.ethoffload = 1;
2909         }
2910
2911         if (sc->toecaps) {
2912                 /* query offload-related parameters */
2913                 param[0] = FW_PARAM_DEV(NTID);
2914                 param[1] = FW_PARAM_PFVF(SERVER_START);
2915                 param[2] = FW_PARAM_PFVF(SERVER_END);
2916                 param[3] = FW_PARAM_PFVF(TDDP_START);
2917                 param[4] = FW_PARAM_PFVF(TDDP_END);
2918                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2919                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2920                 if (rc != 0) {
2921                         device_printf(sc->dev,
2922                             "failed to query TOE parameters: %d.\n", rc);
2923                         return (rc);
2924                 }
2925                 sc->tids.ntids = val[0];
2926                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2927                 sc->tids.stid_base = val[1];
2928                 sc->tids.nstids = val[2] - val[1] + 1;
2929                 sc->vres.ddp.start = val[3];
2930                 sc->vres.ddp.size = val[4] - val[3] + 1;
2931                 sc->params.ofldq_wr_cred = val[5];
2932                 sc->params.offload = 1;
2933         }
2934         if (sc->rdmacaps) {
2935                 param[0] = FW_PARAM_PFVF(STAG_START);
2936                 param[1] = FW_PARAM_PFVF(STAG_END);
2937                 param[2] = FW_PARAM_PFVF(RQ_START);
2938                 param[3] = FW_PARAM_PFVF(RQ_END);
2939                 param[4] = FW_PARAM_PFVF(PBL_START);
2940                 param[5] = FW_PARAM_PFVF(PBL_END);
2941                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2942                 if (rc != 0) {
2943                         device_printf(sc->dev,
2944                             "failed to query RDMA parameters(1): %d.\n", rc);
2945                         return (rc);
2946                 }
2947                 sc->vres.stag.start = val[0];
2948                 sc->vres.stag.size = val[1] - val[0] + 1;
2949                 sc->vres.rq.start = val[2];
2950                 sc->vres.rq.size = val[3] - val[2] + 1;
2951                 sc->vres.pbl.start = val[4];
2952                 sc->vres.pbl.size = val[5] - val[4] + 1;
2953
2954                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2955                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2956                 param[2] = FW_PARAM_PFVF(CQ_START);
2957                 param[3] = FW_PARAM_PFVF(CQ_END);
2958                 param[4] = FW_PARAM_PFVF(OCQ_START);
2959                 param[5] = FW_PARAM_PFVF(OCQ_END);
2960                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2961                 if (rc != 0) {
2962                         device_printf(sc->dev,
2963                             "failed to query RDMA parameters(2): %d.\n", rc);
2964                         return (rc);
2965                 }
2966                 sc->vres.qp.start = val[0];
2967                 sc->vres.qp.size = val[1] - val[0] + 1;
2968                 sc->vres.cq.start = val[2];
2969                 sc->vres.cq.size = val[3] - val[2] + 1;
2970                 sc->vres.ocq.start = val[4];
2971                 sc->vres.ocq.size = val[5] - val[4] + 1;
2972         }
2973         if (sc->iscsicaps) {
2974                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2975                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2976                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2977                 if (rc != 0) {
2978                         device_printf(sc->dev,
2979                             "failed to query iSCSI parameters: %d.\n", rc);
2980                         return (rc);
2981                 }
2982                 sc->vres.iscsi.start = val[0];
2983                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2984         }
2985
2986         /*
2987          * We've got the params we wanted to query via the firmware.  Now grab
2988          * some others directly from the chip.
2989          */
2990         rc = t4_read_chip_settings(sc);
2991
2992         return (rc);
2993 }
2994
2995 static int
2996 set_params__post_init(struct adapter *sc)
2997 {
2998         uint32_t param, val;
2999
3000         /* ask for encapsulated CPLs */
3001         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3002         val = 1;
3003         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3004
3005         return (0);
3006 }
3007
3008 #undef FW_PARAM_PFVF
3009 #undef FW_PARAM_DEV
3010
3011 static void
3012 t4_set_desc(struct adapter *sc)
3013 {
3014         char buf[128];
3015         struct adapter_params *p = &sc->params;
3016
3017         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
3018             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
3019             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
3020
3021         device_set_desc_copy(sc->dev, buf);
3022 }
3023
3024 static void
3025 build_medialist(struct port_info *pi, struct ifmedia *media)
3026 {
3027         int m;
3028
3029         PORT_LOCK(pi);
3030
3031         ifmedia_removeall(media);
3032
3033         m = IFM_ETHER | IFM_FDX;
3034
3035         switch(pi->port_type) {
3036         case FW_PORT_TYPE_BT_XFI:
3037         case FW_PORT_TYPE_BT_XAUI:
3038                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
3039                 /* fall through */
3040
3041         case FW_PORT_TYPE_BT_SGMII:
3042                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
3043                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
3044                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
3045                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
3046                 break;
3047
3048         case FW_PORT_TYPE_CX4:
3049                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
3050                 ifmedia_set(media, m | IFM_10G_CX4);
3051                 break;
3052
3053         case FW_PORT_TYPE_QSFP_10G:
3054         case FW_PORT_TYPE_SFP:
3055         case FW_PORT_TYPE_FIBER_XFI:
3056         case FW_PORT_TYPE_FIBER_XAUI:
3057                 switch (pi->mod_type) {
3058
3059                 case FW_PORT_MOD_TYPE_LR:
3060                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
3061                         ifmedia_set(media, m | IFM_10G_LR);
3062                         break;
3063
3064                 case FW_PORT_MOD_TYPE_SR:
3065                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
3066                         ifmedia_set(media, m | IFM_10G_SR);
3067                         break;
3068
3069                 case FW_PORT_MOD_TYPE_LRM:
3070                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
3071                         ifmedia_set(media, m | IFM_10G_LRM);
3072                         break;
3073
3074                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3075                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3076                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
3077                         ifmedia_set(media, m | IFM_10G_TWINAX);
3078                         break;
3079
3080                 case FW_PORT_MOD_TYPE_NONE:
3081                         m &= ~IFM_FDX;
3082                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3083                         ifmedia_set(media, m | IFM_NONE);
3084                         break;
3085
3086                 case FW_PORT_MOD_TYPE_NA:
3087                 case FW_PORT_MOD_TYPE_ER:
3088                 default:
3089                         device_printf(pi->dev,
3090                             "unknown port_type (%d), mod_type (%d)\n",
3091                             pi->port_type, pi->mod_type);
3092                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3093                         ifmedia_set(media, m | IFM_UNKNOWN);
3094                         break;
3095                 }
3096                 break;
3097
3098         case FW_PORT_TYPE_QSFP:
3099                 switch (pi->mod_type) {
3100
3101                 case FW_PORT_MOD_TYPE_LR:
3102                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
3103                         ifmedia_set(media, m | IFM_40G_LR4);
3104                         break;
3105
3106                 case FW_PORT_MOD_TYPE_SR:
3107                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
3108                         ifmedia_set(media, m | IFM_40G_SR4);
3109                         break;
3110
3111                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3112                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3113                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
3114                         ifmedia_set(media, m | IFM_40G_CR4);
3115                         break;
3116
3117                 case FW_PORT_MOD_TYPE_NONE:
3118                         m &= ~IFM_FDX;
3119                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3120                         ifmedia_set(media, m | IFM_NONE);
3121                         break;
3122
3123                 default:
3124                         device_printf(pi->dev,
3125                             "unknown port_type (%d), mod_type (%d)\n",
3126                             pi->port_type, pi->mod_type);
3127                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3128                         ifmedia_set(media, m | IFM_UNKNOWN);
3129                         break;
3130                 }
3131                 break;
3132
3133         default:
3134                 device_printf(pi->dev,
3135                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3136                     pi->mod_type);
3137                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3138                 ifmedia_set(media, m | IFM_UNKNOWN);
3139                 break;
3140         }
3141
3142         PORT_UNLOCK(pi);
3143 }
3144
3145 #define FW_MAC_EXACT_CHUNK      7
3146
3147 /*
3148  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3149  * indicates which parameters should be programmed (the rest are left alone).
3150  */
3151 int
3152 update_mac_settings(struct ifnet *ifp, int flags)
3153 {
3154         int rc = 0;
3155         struct vi_info *vi = ifp->if_softc;
3156         struct port_info *pi = vi->pi;
3157         struct adapter *sc = pi->adapter;
3158         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3159
3160         ASSERT_SYNCHRONIZED_OP(sc);
3161         KASSERT(flags, ("%s: not told what to update.", __func__));
3162
3163         if (flags & XGMAC_MTU)
3164                 mtu = ifp->if_mtu;
3165
3166         if (flags & XGMAC_PROMISC)
3167                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3168
3169         if (flags & XGMAC_ALLMULTI)
3170                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3171
3172         if (flags & XGMAC_VLANEX)
3173                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3174
3175         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3176                 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
3177                     allmulti, 1, vlanex, false);
3178                 if (rc) {
3179                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3180                             rc);
3181                         return (rc);
3182                 }
3183         }
3184
3185         if (flags & XGMAC_UCADDR) {
3186                 uint8_t ucaddr[ETHER_ADDR_LEN];
3187
3188                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3189                 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
3190                     ucaddr, true, true);
3191                 if (rc < 0) {
3192                         rc = -rc;
3193                         if_printf(ifp, "change_mac failed: %d\n", rc);
3194                         return (rc);
3195                 } else {
3196                         vi->xact_addr_filt = rc;
3197                         rc = 0;
3198                 }
3199         }
3200
3201         if (flags & XGMAC_MCADDRS) {
3202                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3203                 int del = 1;
3204                 uint64_t hash = 0;
3205                 struct ifmultiaddr *ifma;
3206                 int i = 0, j;
3207
3208                 if_maddr_rlock(ifp);
3209                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3210                         if (ifma->ifma_addr->sa_family != AF_LINK)
3211                                 continue;
3212                         mcaddr[i] =
3213                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3214                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3215                         i++;
3216
3217                         if (i == FW_MAC_EXACT_CHUNK) {
3218                                 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
3219                                     del, i, mcaddr, NULL, &hash, 0);
3220                                 if (rc < 0) {
3221                                         rc = -rc;
3222                                         for (j = 0; j < i; j++) {
3223                                                 if_printf(ifp,
3224                                                     "failed to add mc address"
3225                                                     " %02x:%02x:%02x:"
3226                                                     "%02x:%02x:%02x rc=%d\n",
3227                                                     mcaddr[j][0], mcaddr[j][1],
3228                                                     mcaddr[j][2], mcaddr[j][3],
3229                                                     mcaddr[j][4], mcaddr[j][5],
3230                                                     rc);
3231                                         }
3232                                         goto mcfail;
3233                                 }
3234                                 del = 0;
3235                                 i = 0;
3236                         }
3237                 }
3238                 if (i > 0) {
3239                         rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
3240                             mcaddr, NULL, &hash, 0);
3241                         if (rc < 0) {
3242                                 rc = -rc;
3243                                 for (j = 0; j < i; j++) {
3244                                         if_printf(ifp,
3245                                             "failed to add mc address"
3246                                             " %02x:%02x:%02x:"
3247                                             "%02x:%02x:%02x rc=%d\n",
3248                                             mcaddr[j][0], mcaddr[j][1],
3249                                             mcaddr[j][2], mcaddr[j][3],
3250                                             mcaddr[j][4], mcaddr[j][5],
3251                                             rc);
3252                                 }
3253                                 goto mcfail;
3254                         }
3255                 }
3256
3257                 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
3258                 if (rc != 0)
3259                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3260 mcfail:
3261                 if_maddr_runlock(ifp);
3262         }
3263
3264         return (rc);
3265 }
3266
3267 /*
3268  * {begin|end}_synchronized_op must be called from the same thread.
3269  */
3270 int
3271 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
3272     char *wmesg)
3273 {
3274         int rc, pri;
3275
3276 #ifdef WITNESS
3277         /* the caller thinks it's ok to sleep, but is it really? */
3278         if (flags & SLEEP_OK)
3279                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
3280                     "begin_synchronized_op");
3281 #endif
3282
3283         if (INTR_OK)
3284                 pri = PCATCH;
3285         else
3286                 pri = 0;
3287
3288         ADAPTER_LOCK(sc);
3289         for (;;) {
3290
3291                 if (vi && IS_DOOMED(vi)) {
3292                         rc = ENXIO;
3293                         goto done;
3294                 }
3295
3296                 if (!IS_BUSY(sc)) {
3297                         rc = 0;
3298                         break;
3299                 }
3300
3301                 if (!(flags & SLEEP_OK)) {
3302                         rc = EBUSY;
3303                         goto done;
3304                 }
3305
3306                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3307                         rc = EINTR;
3308                         goto done;
3309                 }
3310         }
3311
3312         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3313         SET_BUSY(sc);
3314 #ifdef INVARIANTS
3315         sc->last_op = wmesg;
3316         sc->last_op_thr = curthread;
3317         sc->last_op_flags = flags;
3318 #endif
3319
3320 done:
3321         if (!(flags & HOLD_LOCK) || rc)
3322                 ADAPTER_UNLOCK(sc);
3323
3324         return (rc);
3325 }
3326
3327 /*
3328  * Tell if_ioctl and if_init that the VI is going away.  This is
3329  * special variant of begin_synchronized_op and must be paired with a
3330  * call to end_synchronized_op.
3331  */
3332 void
3333 doom_vi(struct adapter *sc, struct vi_info *vi)
3334 {
3335
3336         ADAPTER_LOCK(sc);
3337         SET_DOOMED(vi);
3338         wakeup(&sc->flags);
3339         while (IS_BUSY(sc))
3340                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
3341         SET_BUSY(sc);
3342 #ifdef INVARIANTS
3343         sc->last_op = "t4detach";
3344         sc->last_op_thr = curthread;
3345         sc->last_op_flags = 0;
3346 #endif
3347         ADAPTER_UNLOCK(sc);
3348 }
3349
3350 /*
3351  * {begin|end}_synchronized_op must be called from the same thread.
3352  */
3353 void
3354 end_synchronized_op(struct adapter *sc, int flags)
3355 {
3356
3357         if (flags & LOCK_HELD)
3358                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3359         else
3360                 ADAPTER_LOCK(sc);
3361
3362         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3363         CLR_BUSY(sc);
3364         wakeup(&sc->flags);
3365         ADAPTER_UNLOCK(sc);
3366 }
3367
3368 static int
3369 cxgbe_init_synchronized(struct vi_info *vi)
3370 {
3371         struct port_info *pi = vi->pi;
3372         struct adapter *sc = pi->adapter;
3373         struct ifnet *ifp = vi->ifp;
3374         int rc = 0, i;
3375         struct sge_txq *txq;
3376
3377         ASSERT_SYNCHRONIZED_OP(sc);
3378
3379         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3380                 return (0);     /* already running */
3381
3382         if (!(sc->flags & FULL_INIT_DONE) &&
3383             ((rc = adapter_full_init(sc)) != 0))
3384                 return (rc);    /* error message displayed already */
3385
3386         if (!(vi->flags & VI_INIT_DONE) &&
3387             ((rc = vi_full_init(vi)) != 0))
3388                 return (rc); /* error message displayed already */
3389
3390         rc = update_mac_settings(ifp, XGMAC_ALL);
3391         if (rc)
3392                 goto done;      /* error message displayed already */
3393
3394         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
3395         if (rc != 0) {
3396                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3397                 goto done;
3398         }
3399
3400         /*
3401          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3402          * if this changes.
3403          */
3404
3405         for_each_txq(vi, i, txq) {
3406                 TXQ_LOCK(txq);
3407                 txq->eq.flags |= EQ_ENABLED;
3408                 TXQ_UNLOCK(txq);
3409         }
3410
3411         /*
3412          * The first iq of the first port to come up is used for tracing.
3413          */
3414         if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
3415                 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
3416                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3417                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3418                     V_QUEUENUMBER(sc->traceq));
3419                 pi->flags |= HAS_TRACEQ;
3420         }
3421
3422         /* all ok */
3423         PORT_LOCK(pi);
3424         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3425         pi->up_vis++;
3426
3427         if (pi->nvi > 1)
3428                 callout_reset(&vi->tick, hz, vi_tick, vi);
3429         else
3430                 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3431         PORT_UNLOCK(pi);
3432 done:
3433         if (rc != 0)
3434                 cxgbe_uninit_synchronized(vi);
3435
3436         return (rc);
3437 }
3438
3439 /*
3440  * Idempotent.
3441  */
3442 static int
3443 cxgbe_uninit_synchronized(struct vi_info *vi)
3444 {
3445         struct port_info *pi = vi->pi;
3446         struct adapter *sc = pi->adapter;
3447         struct ifnet *ifp = vi->ifp;
3448         int rc, i;
3449         struct sge_txq *txq;
3450
3451         ASSERT_SYNCHRONIZED_OP(sc);
3452
3453         if (!(vi->flags & VI_INIT_DONE)) {
3454                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3455                     ("uninited VI is running"));
3456                 return (0);
3457         }
3458
3459         /*
3460          * Disable the VI so that all its data in either direction is discarded
3461          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3462          * tick) intact as the TP can deliver negative advice or data that it's
3463          * holding in its RAM (for an offloaded connection) even after the VI is
3464          * disabled.
3465          */
3466         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
3467         if (rc) {
3468                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3469                 return (rc);
3470         }
3471
3472         for_each_txq(vi, i, txq) {
3473                 TXQ_LOCK(txq);
3474                 txq->eq.flags &= ~EQ_ENABLED;
3475                 TXQ_UNLOCK(txq);
3476         }
3477
3478         PORT_LOCK(pi);
3479         if (pi->nvi == 1)
3480                 callout_stop(&pi->tick);
3481         else
3482                 callout_stop(&vi->tick);
3483         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3484                 PORT_UNLOCK(pi);
3485                 return (0);
3486         }
3487         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3488         pi->up_vis--;
3489         if (pi->up_vis > 0) {
3490                 PORT_UNLOCK(pi);
3491                 return (0);
3492         }
3493         PORT_UNLOCK(pi);
3494
3495         pi->link_cfg.link_ok = 0;
3496         pi->link_cfg.speed = 0;
3497         pi->linkdnrc = -1;
3498         t4_os_link_changed(sc, pi->port_id, 0, -1);
3499
3500         return (0);
3501 }
3502
3503 /*
3504  * It is ok for this function to fail midway and return right away.  t4_detach
3505  * will walk the entire sc->irq list and clean up whatever is valid.
3506  */
3507 static int
3508 setup_intr_handlers(struct adapter *sc)
3509 {
3510         int rc, rid, p, q, v;
3511         char s[8];
3512         struct irq *irq;
3513         struct port_info *pi;
3514         struct vi_info *vi;
3515         struct sge *sge = &sc->sge;
3516         struct sge_rxq *rxq;
3517 #ifdef TCP_OFFLOAD
3518         struct sge_ofld_rxq *ofld_rxq;
3519 #endif
3520 #ifdef DEV_NETMAP
3521         struct sge_nm_rxq *nm_rxq;
3522 #endif
3523
3524         /*
3525          * Setup interrupts.
3526          */
3527         irq = &sc->irq[0];
3528         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3529         if (sc->intr_count == 1)
3530                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3531
3532         /* Multiple interrupts. */
3533         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3534             ("%s: too few intr.", __func__));
3535
3536         /* The first one is always error intr */
3537         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3538         if (rc != 0)
3539                 return (rc);
3540         irq++;
3541         rid++;
3542
3543         /* The second one is always the firmware event queue */
3544         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
3545         if (rc != 0)
3546                 return (rc);
3547         irq++;
3548         rid++;
3549
3550         for_each_port(sc, p) {
3551                 pi = sc->port[p];
3552                 for_each_vi(pi, v, vi) {
3553                         vi->first_intr = rid - 1;
3554
3555                         if (vi->nnmrxq > 0) {
3556                                 int n = max(vi->nrxq, vi->nnmrxq);
3557
3558                                 MPASS(vi->flags & INTR_RXQ);
3559
3560                                 rxq = &sge->rxq[vi->first_rxq];
3561 #ifdef DEV_NETMAP
3562                                 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
3563 #endif
3564                                 for (q = 0; q < n; q++) {
3565                                         snprintf(s, sizeof(s), "%x%c%x", p,
3566                                             'a' + v, q);
3567                                         if (q < vi->nrxq)
3568                                                 irq->rxq = rxq++;
3569 #ifdef DEV_NETMAP
3570                                         if (q < vi->nnmrxq)
3571                                                 irq->nm_rxq = nm_rxq++;
3572 #endif
3573                                         rc = t4_alloc_irq(sc, irq, rid,
3574                                             t4_vi_intr, irq, s);
3575                                         if (rc != 0)
3576                                                 return (rc);
3577                                         irq++;
3578                                         rid++;
3579                                         vi->nintr++;
3580                                 }
3581                         } else if (vi->flags & INTR_RXQ) {
3582                                 for_each_rxq(vi, q, rxq) {
3583                                         snprintf(s, sizeof(s), "%x%c%x", p,
3584                                             'a' + v, q);
3585                                         rc = t4_alloc_irq(sc, irq, rid,
3586                                             t4_intr, rxq, s);
3587                                         if (rc != 0)
3588                                                 return (rc);
3589                                         irq++;
3590                                         rid++;
3591                                         vi->nintr++;
3592                                 }
3593                         }
3594 #ifdef TCP_OFFLOAD
3595                         if (vi->flags & INTR_OFLD_RXQ) {
3596                                 for_each_ofld_rxq(vi, q, ofld_rxq) {
3597                                         snprintf(s, sizeof(s), "%x%c%x", p,
3598                                             'A' + v, q);
3599                                         rc = t4_alloc_irq(sc, irq, rid,
3600                                             t4_intr, ofld_rxq, s);
3601                                         if (rc != 0)
3602                                                 return (rc);
3603                                         irq++;
3604                                         rid++;
3605                                         vi->nintr++;
3606                                 }
3607                         }
3608 #endif
3609                 }
3610         }
3611         MPASS(irq == &sc->irq[sc->intr_count]);
3612
3613         return (0);
3614 }
3615
3616 int
3617 adapter_full_init(struct adapter *sc)
3618 {
3619         int rc, i;
3620
3621         ASSERT_SYNCHRONIZED_OP(sc);
3622         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3623         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3624             ("%s: FULL_INIT_DONE already", __func__));
3625
3626         /*
3627          * queues that belong to the adapter (not any particular port).
3628          */
3629         rc = t4_setup_adapter_queues(sc);
3630         if (rc != 0)
3631                 goto done;
3632
3633         for (i = 0; i < nitems(sc->tq); i++) {
3634                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3635                     taskqueue_thread_enqueue, &sc->tq[i]);
3636                 if (sc->tq[i] == NULL) {
3637                         device_printf(sc->dev,
3638                             "failed to allocate task queue %d\n", i);
3639                         rc = ENOMEM;
3640                         goto done;
3641                 }
3642                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3643                     device_get_nameunit(sc->dev), i);
3644         }
3645
3646         t4_intr_enable(sc);
3647         sc->flags |= FULL_INIT_DONE;
3648 done:
3649         if (rc != 0)
3650                 adapter_full_uninit(sc);
3651
3652         return (rc);
3653 }
3654
3655 int
3656 adapter_full_uninit(struct adapter *sc)
3657 {
3658         int i;
3659
3660         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3661
3662         t4_teardown_adapter_queues(sc);
3663
3664         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3665                 taskqueue_free(sc->tq[i]);
3666                 sc->tq[i] = NULL;
3667         }
3668
3669         sc->flags &= ~FULL_INIT_DONE;
3670
3671         return (0);
3672 }
3673
3674 #ifdef RSS
3675 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
3676     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
3677     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
3678     RSS_HASHTYPE_RSS_UDP_IPV6)
3679
3680 /* Translates kernel hash types to hardware. */
3681 static int
3682 hashconfig_to_hashen(int hashconfig)
3683 {
3684         int hashen = 0;
3685
3686         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
3687                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
3688         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
3689                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
3690         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
3691                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3692                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3693         }
3694         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
3695                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3696                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3697         }
3698         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
3699                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3700         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
3701                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3702
3703         return (hashen);
3704 }
3705
3706 /* Translates hardware hash types to kernel. */
3707 static int
3708 hashen_to_hashconfig(int hashen)
3709 {
3710         int hashconfig = 0;
3711
3712         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
3713                 /*
3714                  * If UDP hashing was enabled it must have been enabled for
3715                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
3716                  * enabling any 4-tuple hash is nonsense configuration.
3717                  */
3718                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3719                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
3720
3721                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3722                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
3723                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3724                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
3725         }
3726         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3727                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
3728         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3729                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
3730         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
3731                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
3732         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
3733                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
3734
3735         return (hashconfig);
3736 }
3737 #endif
3738
3739 int
3740 vi_full_init(struct vi_info *vi)
3741 {
3742         struct adapter *sc = vi->pi->adapter;
3743         struct ifnet *ifp = vi->ifp;
3744         uint16_t *rss;
3745         struct sge_rxq *rxq;
3746         int rc, i, j, hashen;
3747 #ifdef RSS
3748         int nbuckets = rss_getnumbuckets();
3749         int hashconfig = rss_gethashconfig();
3750         int extra;
3751         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3752         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3753 #endif
3754
3755         ASSERT_SYNCHRONIZED_OP(sc);
3756         KASSERT((vi->flags & VI_INIT_DONE) == 0,
3757             ("%s: VI_INIT_DONE already", __func__));
3758
3759         sysctl_ctx_init(&vi->ctx);
3760         vi->flags |= VI_SYSCTL_CTX;
3761
3762         /*
3763          * Allocate tx/rx/fl queues for this VI.
3764          */
3765         rc = t4_setup_vi_queues(vi);
3766         if (rc != 0)
3767                 goto done;      /* error message displayed already */
3768
3769         /*
3770          * Setup RSS for this VI.  Save a copy of the RSS table for later use.
3771          */
3772         if (vi->nrxq > vi->rss_size) {
3773                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
3774                     "some queues will never receive traffic.\n", vi->nrxq,
3775                     vi->rss_size);
3776         } else if (vi->rss_size % vi->nrxq) {
3777                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
3778                     "expect uneven traffic distribution.\n", vi->nrxq,
3779                     vi->rss_size);
3780         }
3781 #ifdef RSS
3782         MPASS(RSS_KEYSIZE == 40);
3783         if (vi->nrxq != nbuckets) {
3784                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
3785                     "performance will be impacted.\n", vi->nrxq, nbuckets);
3786         }
3787
3788         rss_getkey((void *)&raw_rss_key[0]);
3789         for (i = 0; i < nitems(rss_key); i++) {
3790                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
3791         }
3792         t4_write_rss_key(sc, (void *)&rss_key[0], -1);
3793 #endif
3794         rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3795         for (i = 0; i < vi->rss_size;) {
3796 #ifdef RSS
3797                 j = rss_get_indirection_to_bucket(i);
3798                 j %= vi->nrxq;
3799                 rxq = &sc->sge.rxq[vi->first_rxq + j];
3800                 rss[i++] = rxq->iq.abs_id;
3801 #else
3802                 for_each_rxq(vi, j, rxq) {
3803                         rss[i++] = rxq->iq.abs_id;
3804                         if (i == vi->rss_size)
3805                                 break;
3806                 }
3807 #endif
3808         }
3809
3810         rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
3811             vi->rss_size);
3812         if (rc != 0) {
3813                 if_printf(ifp, "rss_config failed: %d\n", rc);
3814                 goto done;
3815         }
3816
3817 #ifdef RSS
3818         hashen = hashconfig_to_hashen(hashconfig);
3819
3820         /*
3821          * We may have had to enable some hashes even though the global config
3822          * wants them disabled.  This is a potential problem that must be
3823          * reported to the user.
3824          */
3825         extra = hashen_to_hashconfig(hashen) ^ hashconfig;
3826
3827         /*
3828          * If we consider only the supported hash types, then the enabled hashes
3829          * are a superset of the requested hashes.  In other words, there cannot
3830          * be any supported hash that was requested but not enabled, but there
3831          * can be hashes that were not requested but had to be enabled.
3832          */
3833         extra &= SUPPORTED_RSS_HASHTYPES;
3834         MPASS((extra & hashconfig) == 0);
3835
3836         if (extra) {
3837                 if_printf(ifp,
3838                     "global RSS config (0x%x) cannot be accomodated.\n",
3839                     hashconfig);
3840         }
3841         if (extra & RSS_HASHTYPE_RSS_IPV4)
3842                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
3843         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
3844                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
3845         if (extra & RSS_HASHTYPE_RSS_IPV6)
3846                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
3847         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
3848                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
3849         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
3850                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
3851         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
3852                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
3853 #else
3854         hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
3855             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
3856             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3857             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
3858 #endif
3859         rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]);
3860         if (rc != 0) {
3861                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
3862                 goto done;
3863         }
3864
3865         vi->rss = rss;
3866         vi->flags |= VI_INIT_DONE;
3867 done:
3868         if (rc != 0)
3869                 vi_full_uninit(vi);
3870
3871         return (rc);
3872 }
3873
3874 /*
3875  * Idempotent.
3876  */
3877 int
3878 vi_full_uninit(struct vi_info *vi)
3879 {
3880         struct port_info *pi = vi->pi;
3881         struct adapter *sc = pi->adapter;
3882         int i;
3883         struct sge_rxq *rxq;
3884         struct sge_txq *txq;
3885 #ifdef TCP_OFFLOAD
3886         struct sge_ofld_rxq *ofld_rxq;
3887         struct sge_wrq *ofld_txq;
3888 #endif
3889
3890         if (vi->flags & VI_INIT_DONE) {
3891
3892                 /* Need to quiesce queues.  */
3893
3894                 /* XXX: Only for the first VI? */
3895                 if (IS_MAIN_VI(vi))
3896                         quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3897
3898                 for_each_txq(vi, i, txq) {
3899                         quiesce_txq(sc, txq);
3900                 }
3901
3902 #ifdef TCP_OFFLOAD
3903                 for_each_ofld_txq(vi, i, ofld_txq) {
3904                         quiesce_wrq(sc, ofld_txq);
3905                 }
3906 #endif
3907
3908                 for_each_rxq(vi, i, rxq) {
3909                         quiesce_iq(sc, &rxq->iq);
3910                         quiesce_fl(sc, &rxq->fl);
3911                 }
3912
3913 #ifdef TCP_OFFLOAD
3914                 for_each_ofld_rxq(vi, i, ofld_rxq) {
3915                         quiesce_iq(sc, &ofld_rxq->iq);
3916                         quiesce_fl(sc, &ofld_rxq->fl);
3917                 }
3918 #endif
3919                 free(vi->rss, M_CXGBE);
3920                 free(vi->nm_rss, M_CXGBE);
3921         }
3922
3923         t4_teardown_vi_queues(vi);
3924         vi->flags &= ~VI_INIT_DONE;
3925
3926         return (0);
3927 }
3928
3929 static void
3930 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3931 {
3932         struct sge_eq *eq = &txq->eq;
3933         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3934
3935         (void) sc;      /* unused */
3936
3937 #ifdef INVARIANTS
3938         TXQ_LOCK(txq);
3939         MPASS((eq->flags & EQ_ENABLED) == 0);
3940         TXQ_UNLOCK(txq);
3941 #endif
3942
3943         /* Wait for the mp_ring to empty. */
3944         while (!mp_ring_is_idle(txq->r)) {
3945                 mp_ring_check_drainage(txq->r, 0);
3946                 pause("rquiesce", 1);
3947         }
3948
3949         /* Then wait for the hardware to finish. */
3950         while (spg->cidx != htobe16(eq->pidx))
3951                 pause("equiesce", 1);
3952
3953         /* Finally, wait for the driver to reclaim all descriptors. */
3954         while (eq->cidx != eq->pidx)
3955                 pause("dquiesce", 1);
3956 }
3957
3958 static void
3959 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3960 {
3961
3962         /* XXXTX */
3963 }
3964
3965 static void
3966 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3967 {
3968         (void) sc;      /* unused */
3969
3970         /* Synchronize with the interrupt handler */
3971         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3972                 pause("iqfree", 1);
3973 }
3974
3975 static void
3976 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3977 {
3978         mtx_lock(&sc->sfl_lock);
3979         FL_LOCK(fl);
3980         fl->flags |= FL_DOOMED;
3981         FL_UNLOCK(fl);
3982         callout_stop(&sc->sfl_callout);
3983         mtx_unlock(&sc->sfl_lock);
3984
3985         KASSERT((fl->flags & FL_STARVING) == 0,
3986             ("%s: still starving", __func__));
3987 }
3988
3989 static int
3990 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3991     driver_intr_t *handler, void *arg, char *name)
3992 {
3993         int rc;
3994
3995         irq->rid = rid;
3996         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3997             RF_SHAREABLE | RF_ACTIVE);
3998         if (irq->res == NULL) {
3999                 device_printf(sc->dev,
4000                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
4001                 return (ENOMEM);
4002         }
4003
4004         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
4005             NULL, handler, arg, &irq->tag);
4006         if (rc != 0) {
4007                 device_printf(sc->dev,
4008                     "failed to setup interrupt for rid %d, name %s: %d\n",
4009                     rid, name, rc);
4010         } else if (name)
4011                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
4012
4013         return (rc);
4014 }
4015
4016 static int
4017 t4_free_irq(struct adapter *sc, struct irq *irq)
4018 {
4019         if (irq->tag)
4020                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
4021         if (irq->res)
4022                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
4023
4024         bzero(irq, sizeof(*irq));
4025
4026         return (0);
4027 }
4028
4029 static void
4030 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
4031     unsigned int end)
4032 {
4033         uint32_t *p = (uint32_t *)(buf + start);
4034
4035         for ( ; start <= end; start += sizeof(uint32_t))
4036                 *p++ = t4_read_reg(sc, start);
4037 }
4038
4039 static void
4040 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
4041 {
4042         int i, n;
4043         const unsigned int *reg_ranges;
4044         static const unsigned int t4_reg_ranges[] = {
4045                 0x1008, 0x1108,
4046                 0x1180, 0x11b4,
4047                 0x11fc, 0x123c,
4048                 0x1300, 0x173c,
4049                 0x1800, 0x18fc,
4050                 0x3000, 0x30d8,
4051                 0x30e0, 0x5924,
4052                 0x5960, 0x59d4,
4053                 0x5a00, 0x5af8,
4054                 0x6000, 0x6098,
4055                 0x6100, 0x6150,
4056                 0x6200, 0x6208,
4057                 0x6240, 0x6248,
4058                 0x6280, 0x6338,
4059                 0x6370, 0x638c,
4060                 0x6400, 0x643c,
4061                 0x6500, 0x6524,
4062                 0x6a00, 0x6a38,
4063                 0x6a60, 0x6a78,
4064                 0x6b00, 0x6b84,
4065                 0x6bf0, 0x6c84,
4066                 0x6cf0, 0x6d84,
4067                 0x6df0, 0x6e84,
4068                 0x6ef0, 0x6f84,
4069                 0x6ff0, 0x7084,
4070                 0x70f0, 0x7184,
4071                 0x71f0, 0x7284,
4072                 0x72f0, 0x7384,
4073                 0x73f0, 0x7450,
4074                 0x7500, 0x7530,
4075                 0x7600, 0x761c,
4076                 0x7680, 0x76cc,
4077                 0x7700, 0x7798,
4078                 0x77c0, 0x77fc,
4079                 0x7900, 0x79fc,
4080                 0x7b00, 0x7c38,
4081                 0x7d00, 0x7efc,
4082                 0x8dc0, 0x8e1c,
4083                 0x8e30, 0x8e78,
4084                 0x8ea0, 0x8f6c,
4085                 0x8fc0, 0x9074,
4086                 0x90fc, 0x90fc,
4087                 0x9400, 0x9458,
4088                 0x9600, 0x96bc,
4089                 0x9800, 0x9808,
4090                 0x9820, 0x983c,
4091                 0x9850, 0x9864,
4092                 0x9c00, 0x9c6c,
4093                 0x9c80, 0x9cec,
4094                 0x9d00, 0x9d6c,
4095                 0x9d80, 0x9dec,
4096                 0x9e00, 0x9e6c,
4097                 0x9e80, 0x9eec,
4098                 0x9f00, 0x9f6c,
4099                 0x9f80, 0x9fec,
4100                 0xd004, 0xd03c,
4101                 0xdfc0, 0xdfe0,
4102                 0xe000, 0xea7c,
4103                 0xf000, 0x11110,
4104                 0x11118, 0x11190,
4105                 0x19040, 0x1906c,
4106                 0x19078, 0x19080,
4107                 0x1908c, 0x19124,
4108                 0x19150, 0x191b0,
4109                 0x191d0, 0x191e8,
4110                 0x19238, 0x1924c,
4111                 0x193f8, 0x19474,
4112                 0x19490, 0x194f8,
4113                 0x19800, 0x19f30,
4114                 0x1a000, 0x1a06c,
4115                 0x1a0b0, 0x1a120,
4116                 0x1a128, 0x1a138,
4117                 0x1a190, 0x1a1c4,
4118                 0x1a1fc, 0x1a1fc,
4119                 0x1e040, 0x1e04c,
4120                 0x1e284, 0x1e28c,
4121                 0x1e2c0, 0x1e2c0,
4122                 0x1e2e0, 0x1e2e0,
4123                 0x1e300, 0x1e384,
4124                 0x1e3c0, 0x1e3c8,
4125                 0x1e440, 0x1e44c,
4126                 0x1e684, 0x1e68c,
4127                 0x1e6c0, 0x1e6c0,
4128                 0x1e6e0, 0x1e6e0,
4129                 0x1e700, 0x1e784,
4130                 0x1e7c0, 0x1e7c8,
4131                 0x1e840, 0x1e84c,
4132                 0x1ea84, 0x1ea8c,
4133                 0x1eac0, 0x1eac0,
4134                 0x1eae0, 0x1eae0,
4135                 0x1eb00, 0x1eb84,
4136                 0x1ebc0, 0x1ebc8,
4137                 0x1ec40, 0x1ec4c,
4138                 0x1ee84, 0x1ee8c,
4139                 0x1eec0, 0x1eec0,
4140                 0x1eee0, 0x1eee0,
4141                 0x1ef00, 0x1ef84,
4142                 0x1efc0, 0x1efc8,
4143                 0x1f040, 0x1f04c,
4144                 0x1f284, 0x1f28c,
4145                 0x1f2c0, 0x1f2c0,
4146                 0x1f2e0, 0x1f2e0,
4147                 0x1f300, 0x1f384,
4148                 0x1f3c0, 0x1f3c8,
4149                 0x1f440, 0x1f44c,
4150                 0x1f684, 0x1f68c,
4151                 0x1f6c0, 0x1f6c0,
4152                 0x1f6e0, 0x1f6e0,
4153                 0x1f700, 0x1f784,
4154                 0x1f7c0, 0x1f7c8,
4155                 0x1f840, 0x1f84c,
4156                 0x1fa84, 0x1fa8c,
4157                 0x1fac0, 0x1fac0,
4158                 0x1fae0, 0x1fae0,
4159                 0x1fb00, 0x1fb84,
4160                 0x1fbc0, 0x1fbc8,
4161                 0x1fc40, 0x1fc4c,
4162                 0x1fe84, 0x1fe8c,
4163                 0x1fec0, 0x1fec0,
4164                 0x1fee0, 0x1fee0,
4165                 0x1ff00, 0x1ff84,
4166                 0x1ffc0, 0x1ffc8,
4167                 0x20000, 0x2002c,
4168                 0x20100, 0x2013c,
4169                 0x20190, 0x201c8,
4170                 0x20200, 0x20318,
4171                 0x20400, 0x20528,
4172                 0x20540, 0x20614,
4173                 0x21000, 0x21040,
4174                 0x2104c, 0x21060,
4175                 0x210c0, 0x210ec,
4176                 0x21200, 0x21268,
4177                 0x21270, 0x21284,
4178                 0x212fc, 0x21388,
4179                 0x21400, 0x21404,
4180                 0x21500, 0x21518,
4181                 0x2152c, 0x2153c,
4182                 0x21550, 0x21554,
4183                 0x21600, 0x21600,
4184                 0x21608, 0x21628,
4185                 0x21630, 0x2163c,
4186                 0x21700, 0x2171c,
4187                 0x21780, 0x2178c,
4188                 0x21800, 0x21c38,
4189                 0x21c80, 0x21d7c,
4190                 0x21e00, 0x21e04,
4191                 0x22000, 0x2202c,
4192                 0x22100, 0x2213c,
4193                 0x22190, 0x221c8,
4194                 0x22200, 0x22318,
4195                 0x22400, 0x22528,
4196                 0x22540, 0x22614,
4197                 0x23000, 0x23040,
4198                 0x2304c, 0x23060,
4199                 0x230c0, 0x230ec,
4200                 0x23200, 0x23268,
4201                 0x23270, 0x23284,
4202                 0x232fc, 0x23388,
4203                 0x23400, 0x23404,
4204                 0x23500, 0x23518,
4205                 0x2352c, 0x2353c,
4206                 0x23550, 0x23554,
4207                 0x23600, 0x23600,
4208                 0x23608, 0x23628,
4209                 0x23630, 0x2363c,
4210                 0x23700, 0x2371c,
4211                 0x23780, 0x2378c,
4212                 0x23800, 0x23c38,
4213                 0x23c80, 0x23d7c,
4214                 0x23e00, 0x23e04,
4215                 0x24000, 0x2402c,
4216                 0x24100, 0x2413c,
4217                 0x24190, 0x241c8,
4218                 0x24200, 0x24318,
4219                 0x24400, 0x24528,
4220                 0x24540, 0x24614,
4221                 0x25000, 0x25040,
4222                 0x2504c, 0x25060,
4223                 0x250c0, 0x250ec,
4224                 0x25200, 0x25268,
4225                 0x25270, 0x25284,
4226                 0x252fc, 0x25388,
4227                 0x25400, 0x25404,
4228                 0x25500, 0x25518,
4229                 0x2552c, 0x2553c,
4230                 0x25550, 0x25554,
4231                 0x25600, 0x25600,
4232                 0x25608, 0x25628,
4233                 0x25630, 0x2563c,
4234                 0x25700, 0x2571c,
4235                 0x25780, 0x2578c,
4236                 0x25800, 0x25c38,
4237                 0x25c80, 0x25d7c,
4238                 0x25e00, 0x25e04,
4239                 0x26000, 0x2602c,
4240                 0x26100, 0x2613c,
4241                 0x26190, 0x261c8,
4242                 0x26200, 0x26318,
4243                 0x26400, 0x26528,
4244                 0x26540, 0x26614,
4245                 0x27000, 0x27040,
4246                 0x2704c, 0x27060,
4247                 0x270c0, 0x270ec,
4248                 0x27200, 0x27268,
4249                 0x27270, 0x27284,
4250                 0x272fc, 0x27388,
4251                 0x27400, 0x27404,
4252                 0x27500, 0x27518,
4253                 0x2752c, 0x2753c,
4254                 0x27550, 0x27554,
4255                 0x27600, 0x27600,
4256                 0x27608, 0x27628,
4257                 0x27630, 0x2763c,
4258                 0x27700, 0x2771c,
4259                 0x27780, 0x2778c,
4260                 0x27800, 0x27c38,
4261                 0x27c80, 0x27d7c,
4262                 0x27e00, 0x27e04
4263         };
4264         static const unsigned int t5_reg_ranges[] = {
4265                 0x1008, 0x1148,
4266                 0x1180, 0x11b4,
4267                 0x11fc, 0x123c,
4268                 0x1280, 0x173c,
4269                 0x1800, 0x18fc,
4270                 0x3000, 0x3028,
4271                 0x3060, 0x30d8,
4272                 0x30e0, 0x30fc,
4273                 0x3140, 0x357c,
4274                 0x35a8, 0x35cc,
4275                 0x35ec, 0x35ec,
4276                 0x3600, 0x5624,
4277                 0x56cc, 0x575c,
4278                 0x580c, 0x5814,
4279                 0x5890, 0x58bc,
4280                 0x5940, 0x59dc,
4281                 0x59fc, 0x5a18,
4282                 0x5a60, 0x5a9c,
4283                 0x5b94, 0x5bfc,
4284                 0x6000, 0x6040,
4285                 0x6058, 0x614c,
4286                 0x7700, 0x7798,
4287                 0x77c0, 0x78fc,
4288                 0x7b00, 0x7c54,
4289                 0x7d00, 0x7efc,
4290                 0x8dc0, 0x8de0,
4291                 0x8df8, 0x8e84,
4292                 0x8ea0, 0x8f84,
4293                 0x8fc0, 0x90f8,
4294                 0x9400, 0x9470,
4295                 0x9600, 0x96f4,
4296                 0x9800, 0x9808,
4297                 0x9820, 0x983c,
4298                 0x9850, 0x9864,
4299                 0x9c00, 0x9c6c,
4300                 0x9c80, 0x9cec,
4301                 0x9d00, 0x9d6c,
4302                 0x9d80, 0x9dec,
4303                 0x9e00, 0x9e6c,
4304                 0x9e80, 0x9eec,
4305                 0x9f00, 0x9f6c,
4306                 0x9f80, 0xa020,
4307                 0xd004, 0xd03c,
4308                 0xdfc0, 0xdfe0,
4309                 0xe000, 0x11088,
4310                 0x1109c, 0x11110,
4311                 0x11118, 0x1117c,
4312                 0x11190, 0x11204,
4313                 0x19040, 0x1906c,
4314                 0x19078, 0x19080,
4315                 0x1908c, 0x19124,
4316                 0x19150, 0x191b0,
4317                 0x191d0, 0x191e8,
4318                 0x19238, 0x19290,
4319                 0x193f8, 0x19474,
4320                 0x19490, 0x194cc,
4321                 0x194f0, 0x194f8,
4322                 0x19c00, 0x19c60,
4323                 0x19c94, 0x19e10,
4324                 0x19e50, 0x19f34,
4325                 0x19f40, 0x19f50,
4326                 0x19f90, 0x19fe4,
4327                 0x1a000, 0x1a06c,
4328                 0x1a0b0, 0x1a120,
4329                 0x1a128, 0x1a138,
4330                 0x1a190, 0x1a1c4,
4331                 0x1a1fc, 0x1a1fc,
4332                 0x1e008, 0x1e00c,
4333                 0x1e040, 0x1e04c,
4334                 0x1e284, 0x1e290,
4335                 0x1e2c0, 0x1e2c0,
4336                 0x1e2e0, 0x1e2e0,
4337                 0x1e300, 0x1e384,
4338                 0x1e3c0, 0x1e3c8,
4339                 0x1e408, 0x1e40c,
4340                 0x1e440, 0x1e44c,
4341                 0x1e684, 0x1e690,
4342                 0x1e6c0, 0x1e6c0,
4343                 0x1e6e0, 0x1e6e0,
4344                 0x1e700, 0x1e784,
4345                 0x1e7c0, 0x1e7c8,
4346                 0x1e808, 0x1e80c,
4347                 0x1e840, 0x1e84c,
4348                 0x1ea84, 0x1ea90,
4349                 0x1eac0, 0x1eac0,
4350                 0x1eae0, 0x1eae0,
4351                 0x1eb00, 0x1eb84,
4352                 0x1ebc0, 0x1ebc8,
4353                 0x1ec08, 0x1ec0c,
4354                 0x1ec40, 0x1ec4c,
4355                 0x1ee84, 0x1ee90,
4356                 0x1eec0, 0x1eec0,
4357                 0x1eee0, 0x1eee0,
4358                 0x1ef00, 0x1ef84,
4359                 0x1efc0, 0x1efc8,
4360                 0x1f008, 0x1f00c,
4361                 0x1f040, 0x1f04c,
4362                 0x1f284, 0x1f290,
4363                 0x1f2c0, 0x1f2c0,
4364                 0x1f2e0, 0x1f2e0,
4365                 0x1f300, 0x1f384,
4366                 0x1f3c0, 0x1f3c8,
4367                 0x1f408, 0x1f40c,
4368                 0x1f440, 0x1f44c,
4369                 0x1f684, 0x1f690,
4370                 0x1f6c0, 0x1f6c0,
4371                 0x1f6e0, 0x1f6e0,
4372                 0x1f700, 0x1f784,
4373                 0x1f7c0, 0x1f7c8,
4374                 0x1f808, 0x1f80c,
4375                 0x1f840, 0x1f84c,
4376                 0x1fa84, 0x1fa90,
4377                 0x1fac0, 0x1fac0,
4378                 0x1fae0, 0x1fae0,
4379                 0x1fb00, 0x1fb84,
4380                 0x1fbc0, 0x1fbc8,
4381                 0x1fc08, 0x1fc0c,
4382                 0x1fc40, 0x1fc4c,
4383                 0x1fe84, 0x1fe90,
4384                 0x1fec0, 0x1fec0,
4385                 0x1fee0, 0x1fee0,
4386                 0x1ff00, 0x1ff84,
4387                 0x1ffc0, 0x1ffc8,
4388                 0x30000, 0x30030,
4389                 0x30100, 0x30144,
4390                 0x30190, 0x301d0,
4391                 0x30200, 0x30318,
4392                 0x30400, 0x3052c,
4393                 0x30540, 0x3061c,
4394                 0x30800, 0x30834,
4395                 0x308c0, 0x30908,
4396                 0x30910, 0x309ac,
4397                 0x30a00, 0x30a2c,
4398                 0x30a44, 0x30a50,
4399                 0x30a74, 0x30c24,
4400                 0x30d00, 0x30d00,
4401                 0x30d08, 0x30d14,
4402                 0x30d1c, 0x30d20,
4403                 0x30d3c, 0x30d50,
4404                 0x31200, 0x3120c,
4405                 0x31220, 0x31220,
4406                 0x31240, 0x31240,
4407                 0x31600, 0x3160c,
4408                 0x31a00, 0x31a1c,
4409                 0x31e00, 0x31e20,
4410                 0x31e38, 0x31e3c,
4411                 0x31e80, 0x31e80,
4412                 0x31e88, 0x31ea8,
4413                 0x31eb0, 0x31eb4,
4414                 0x31ec8, 0x31ed4,
4415                 0x31fb8, 0x32004,
4416                 0x32200, 0x32200,
4417                 0x32208, 0x32240,
4418                 0x32248, 0x32280,
4419                 0x32288, 0x322c0,
4420                 0x322c8, 0x322fc,
4421                 0x32600, 0x32630,
4422                 0x32a00, 0x32abc,
4423                 0x32b00, 0x32b70,
4424                 0x33000, 0x33048,
4425                 0x33060, 0x3309c,
4426                 0x330f0, 0x33148,
4427                 0x33160, 0x3319c,
4428                 0x331f0, 0x332e4,
4429                 0x332f8, 0x333e4,
4430                 0x333f8, 0x33448,
4431                 0x33460, 0x3349c,
4432                 0x334f0, 0x33548,
4433                 0x33560, 0x3359c,
4434                 0x335f0, 0x336e4,
4435                 0x336f8, 0x337e4,
4436                 0x337f8, 0x337fc,
4437                 0x33814, 0x33814,
4438                 0x3382c, 0x3382c,
4439                 0x33880, 0x3388c,
4440                 0x338e8, 0x338ec,
4441                 0x33900, 0x33948,
4442                 0x33960, 0x3399c,
4443                 0x339f0, 0x33ae4,
4444                 0x33af8, 0x33b10,
4445                 0x33b28, 0x33b28,
4446                 0x33b3c, 0x33b50,
4447                 0x33bf0, 0x33c10,
4448                 0x33c28, 0x33c28,
4449                 0x33c3c, 0x33c50,
4450                 0x33cf0, 0x33cfc,
4451                 0x34000, 0x34030,
4452                 0x34100, 0x34144,
4453                 0x34190, 0x341d0,
4454                 0x34200, 0x34318,
4455                 0x34400, 0x3452c,
4456                 0x34540, 0x3461c,
4457                 0x34800, 0x34834,
4458                 0x348c0, 0x34908,
4459                 0x34910, 0x349ac,
4460                 0x34a00, 0x34a2c,
4461                 0x34a44, 0x34a50,
4462                 0x34a74, 0x34c24,
4463                 0x34d00, 0x34d00,
4464                 0x34d08, 0x34d14,
4465                 0x34d1c, 0x34d20,
4466                 0x34d3c, 0x34d50,
4467                 0x35200, 0x3520c,
4468                 0x35220, 0x35220,
4469                 0x35240, 0x35240,
4470                 0x35600, 0x3560c,
4471                 0x35a00, 0x35a1c,
4472                 0x35e00, 0x35e20,
4473                 0x35e38, 0x35e3c,
4474                 0x35e80, 0x35e80,
4475                 0x35e88, 0x35ea8,
4476                 0x35eb0, 0x35eb4,
4477                 0x35ec8, 0x35ed4,
4478                 0x35fb8, 0x36004,
4479                 0x36200, 0x36200,
4480                 0x36208, 0x36240,
4481                 0x36248, 0x36280,
4482                 0x36288, 0x362c0,
4483                 0x362c8, 0x362fc,
4484                 0x36600, 0x36630,
4485                 0x36a00, 0x36abc,
4486                 0x36b00, 0x36b70,
4487                 0x37000, 0x37048,
4488                 0x37060, 0x3709c,
4489                 0x370f0, 0x37148,
4490                 0x37160, 0x3719c,
4491                 0x371f0, 0x372e4,
4492                 0x372f8, 0x373e4,
4493                 0x373f8, 0x37448,
4494                 0x37460, 0x3749c,
4495                 0x374f0, 0x37548,
4496                 0x37560, 0x3759c,
4497                 0x375f0, 0x376e4,
4498                 0x376f8, 0x377e4,
4499                 0x377f8, 0x377fc,
4500                 0x37814, 0x37814,
4501                 0x3782c, 0x3782c,
4502                 0x37880, 0x3788c,
4503                 0x378e8, 0x378ec,
4504                 0x37900, 0x37948,
4505                 0x37960, 0x3799c,
4506                 0x379f0, 0x37ae4,
4507                 0x37af8, 0x37b10,
4508                 0x37b28, 0x37b28,
4509                 0x37b3c, 0x37b50,
4510                 0x37bf0, 0x37c10,
4511                 0x37c28, 0x37c28,
4512                 0x37c3c, 0x37c50,
4513                 0x37cf0, 0x37cfc,
4514                 0x38000, 0x38030,
4515                 0x38100, 0x38144,
4516                 0x38190, 0x381d0,
4517                 0x38200, 0x38318,
4518                 0x38400, 0x3852c,
4519                 0x38540, 0x3861c,
4520                 0x38800, 0x38834,
4521                 0x388c0, 0x38908,
4522                 0x38910, 0x389ac,
4523                 0x38a00, 0x38a2c,
4524                 0x38a44, 0x38a50,
4525                 0x38a74, 0x38c24,
4526                 0x38d00, 0x38d00,
4527                 0x38d08, 0x38d14,
4528                 0x38d1c, 0x38d20,
4529                 0x38d3c, 0x38d50,
4530                 0x39200, 0x3920c,
4531                 0x39220, 0x39220,
4532                 0x39240, 0x39240,
4533                 0x39600, 0x3960c,
4534                 0x39a00, 0x39a1c,
4535                 0x39e00, 0x39e20,
4536                 0x39e38, 0x39e3c,
4537                 0x39e80, 0x39e80,
4538                 0x39e88, 0x39ea8,
4539                 0x39eb0, 0x39eb4,
4540                 0x39ec8, 0x39ed4,
4541                 0x39fb8, 0x3a004,
4542                 0x3a200, 0x3a200,
4543                 0x3a208, 0x3a240,
4544                 0x3a248, 0x3a280,
4545                 0x3a288, 0x3a2c0,
4546                 0x3a2c8, 0x3a2fc,
4547                 0x3a600, 0x3a630,
4548                 0x3aa00, 0x3aabc,
4549                 0x3ab00, 0x3ab70,
4550                 0x3b000, 0x3b048,
4551                 0x3b060, 0x3b09c,
4552                 0x3b0f0, 0x3b148,
4553                 0x3b160, 0x3b19c,
4554                 0x3b1f0, 0x3b2e4,
4555                 0x3b2f8, 0x3b3e4,
4556                 0x3b3f8, 0x3b448,
4557                 0x3b460, 0x3b49c,
4558                 0x3b4f0, 0x3b548,
4559                 0x3b560, 0x3b59c,
4560                 0x3b5f0, 0x3b6e4,
4561                 0x3b6f8, 0x3b7e4,
4562                 0x3b7f8, 0x3b7fc,
4563                 0x3b814, 0x3b814,
4564                 0x3b82c, 0x3b82c,
4565                 0x3b880, 0x3b88c,
4566                 0x3b8e8, 0x3b8ec,
4567                 0x3b900, 0x3b948,
4568                 0x3b960, 0x3b99c,
4569                 0x3b9f0, 0x3bae4,
4570                 0x3baf8, 0x3bb10,
4571                 0x3bb28, 0x3bb28,
4572                 0x3bb3c, 0x3bb50,
4573                 0x3bbf0, 0x3bc10,
4574                 0x3bc28, 0x3bc28,
4575                 0x3bc3c, 0x3bc50,
4576                 0x3bcf0, 0x3bcfc,
4577                 0x3c000, 0x3c030,
4578                 0x3c100, 0x3c144,
4579                 0x3c190, 0x3c1d0,
4580                 0x3c200, 0x3c318,
4581                 0x3c400, 0x3c52c,
4582                 0x3c540, 0x3c61c,
4583                 0x3c800, 0x3c834,
4584                 0x3c8c0, 0x3c908,
4585                 0x3c910, 0x3c9ac,
4586                 0x3ca00, 0x3ca2c,
4587                 0x3ca44, 0x3ca50,
4588                 0x3ca74, 0x3cc24,
4589                 0x3cd00, 0x3cd00,
4590                 0x3cd08, 0x3cd14,
4591                 0x3cd1c, 0x3cd20,
4592                 0x3cd3c, 0x3cd50,
4593                 0x3d200, 0x3d20c,
4594                 0x3d220, 0x3d220,
4595                 0x3d240, 0x3d240,
4596                 0x3d600, 0x3d60c,
4597                 0x3da00, 0x3da1c,
4598                 0x3de00, 0x3de20,
4599                 0x3de38, 0x3de3c,
4600                 0x3de80, 0x3de80,
4601                 0x3de88, 0x3dea8,
4602                 0x3deb0, 0x3deb4,
4603                 0x3dec8, 0x3ded4,
4604                 0x3dfb8, 0x3e004,
4605                 0x3e200, 0x3e200,
4606                 0x3e208, 0x3e240,
4607                 0x3e248, 0x3e280,
4608                 0x3e288, 0x3e2c0,
4609                 0x3e2c8, 0x3e2fc,
4610                 0x3e600, 0x3e630,
4611                 0x3ea00, 0x3eabc,
4612                 0x3eb00, 0x3eb70,
4613                 0x3f000, 0x3f048,
4614                 0x3f060, 0x3f09c,
4615                 0x3f0f0, 0x3f148,
4616                 0x3f160, 0x3f19c,
4617                 0x3f1f0, 0x3f2e4,
4618                 0x3f2f8, 0x3f3e4,
4619                 0x3f3f8, 0x3f448,
4620                 0x3f460, 0x3f49c,
4621                 0x3f4f0, 0x3f548,
4622                 0x3f560, 0x3f59c,
4623                 0x3f5f0, 0x3f6e4,
4624                 0x3f6f8, 0x3f7e4,
4625                 0x3f7f8, 0x3f7fc,
4626                 0x3f814, 0x3f814,
4627                 0x3f82c, 0x3f82c,
4628                 0x3f880, 0x3f88c,
4629                 0x3f8e8, 0x3f8ec,
4630                 0x3f900, 0x3f948,
4631                 0x3f960, 0x3f99c,
4632                 0x3f9f0, 0x3fae4,
4633                 0x3faf8, 0x3fb10,
4634                 0x3fb28, 0x3fb28,
4635                 0x3fb3c, 0x3fb50,
4636                 0x3fbf0, 0x3fc10,
4637                 0x3fc28, 0x3fc28,
4638                 0x3fc3c, 0x3fc50,
4639                 0x3fcf0, 0x3fcfc,
4640                 0x40000, 0x4000c,
4641                 0x40040, 0x40068,
4642                 0x4007c, 0x40144,
4643                 0x40180, 0x4018c,
4644                 0x40200, 0x40298,
4645                 0x402ac, 0x4033c,
4646                 0x403f8, 0x403fc,
4647                 0x41304, 0x413c4,
4648                 0x41400, 0x4141c,
4649                 0x41480, 0x414d0,
4650                 0x44000, 0x44078,
4651                 0x440c0, 0x44278,
4652                 0x442c0, 0x44478,
4653                 0x444c0, 0x44678,
4654                 0x446c0, 0x44878,
4655                 0x448c0, 0x449fc,
4656                 0x45000, 0x45068,
4657                 0x45080, 0x45084,
4658                 0x450a0, 0x450b0,
4659                 0x45200, 0x45268,
4660                 0x45280, 0x45284,
4661                 0x452a0, 0x452b0,
4662                 0x460c0, 0x460e4,
4663                 0x47000, 0x4708c,
4664                 0x47200, 0x47250,
4665                 0x47400, 0x47420,
4666                 0x47600, 0x47618,
4667                 0x47800, 0x47814,
4668                 0x48000, 0x4800c,
4669                 0x48040, 0x48068,
4670                 0x4807c, 0x48144,
4671                 0x48180, 0x4818c,
4672                 0x48200, 0x48298,
4673                 0x482ac, 0x4833c,
4674                 0x483f8, 0x483fc,
4675                 0x49304, 0x493c4,
4676                 0x49400, 0x4941c,
4677                 0x49480, 0x494d0,
4678                 0x4c000, 0x4c078,
4679                 0x4c0c0, 0x4c278,
4680                 0x4c2c0, 0x4c478,
4681                 0x4c4c0, 0x4c678,
4682                 0x4c6c0, 0x4c878,
4683                 0x4c8c0, 0x4c9fc,
4684                 0x4d000, 0x4d068,
4685                 0x4d080, 0x4d084,
4686                 0x4d0a0, 0x4d0b0,
4687                 0x4d200, 0x4d268,
4688                 0x4d280, 0x4d284,
4689                 0x4d2a0, 0x4d2b0,
4690                 0x4e0c0, 0x4e0e4,
4691                 0x4f000, 0x4f08c,
4692                 0x4f200, 0x4f250,
4693                 0x4f400, 0x4f420,
4694                 0x4f600, 0x4f618,
4695                 0x4f800, 0x4f814,
4696                 0x50000, 0x500cc,
4697                 0x50400, 0x50400,
4698                 0x50800, 0x508cc,
4699                 0x50c00, 0x50c00,
4700                 0x51000, 0x5101c,
4701                 0x51300, 0x51308,
4702         };
4703
4704         if (is_t4(sc)) {
4705                 reg_ranges = &t4_reg_ranges[0];
4706                 n = nitems(t4_reg_ranges);
4707         } else {
4708                 reg_ranges = &t5_reg_ranges[0];
4709                 n = nitems(t5_reg_ranges);
4710         }
4711
4712         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4713         for (i = 0; i < n; i += 2)
4714                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4715 }
4716
4717 #define A_PL_INDIR_CMD  0x1f8
4718
4719 #define S_PL_AUTOINC    31
4720 #define M_PL_AUTOINC    0x1U
4721 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
4722 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
4723
4724 #define S_PL_VFID       20
4725 #define M_PL_VFID       0xffU
4726 #define V_PL_VFID(x)    ((x) << S_PL_VFID)
4727 #define G_PL_VFID(x)    (((x) >> S_PL_VFID) & M_PL_VFID)
4728
4729 #define S_PL_ADDR       0
4730 #define M_PL_ADDR       0xfffffU
4731 #define V_PL_ADDR(x)    ((x) << S_PL_ADDR)
4732 #define G_PL_ADDR(x)    (((x) >> S_PL_ADDR) & M_PL_ADDR)
4733
4734 #define A_PL_INDIR_DATA 0x1fc
4735
4736 static uint64_t
4737 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
4738 {
4739         u32 stats[2];
4740
4741         mtx_assert(&sc->regwin_lock, MA_OWNED);
4742         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4743             V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg)));
4744         stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
4745         stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
4746         return (((uint64_t)stats[1]) << 32 | stats[0]);
4747 }
4748
4749 static void
4750 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
4751     struct fw_vi_stats_vf *stats)
4752 {
4753
4754 #define GET_STAT(name) \
4755         read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
4756
4757         stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
4758         stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
4759         stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
4760         stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
4761         stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
4762         stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
4763         stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
4764         stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
4765         stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
4766         stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
4767         stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
4768         stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
4769         stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
4770         stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
4771         stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
4772         stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
4773
4774 #undef GET_STAT
4775 }
4776
4777 static void
4778 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
4779 {
4780         int reg;
4781
4782         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4783             V_PL_VFID(G_FW_VIID_VIN(viid)) |
4784             V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
4785         for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
4786              reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
4787                 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
4788 }
4789
4790 static void
4791 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
4792 {
4793         struct ifnet *ifp = vi->ifp;
4794         struct sge_txq *txq;
4795         int i, drops;
4796         struct fw_vi_stats_vf *s = &vi->stats;
4797         struct timeval tv;
4798         const struct timeval interval = {0, 250000};    /* 250ms */
4799
4800         if (!(vi->flags & VI_INIT_DONE))
4801                 return;
4802
4803         getmicrotime(&tv);
4804         timevalsub(&tv, &interval);
4805         if (timevalcmp(&tv, &vi->last_refreshed, <))
4806                 return;
4807
4808         mtx_lock(&sc->regwin_lock);
4809         t4_get_vi_stats(sc, vi->viid, &vi->stats);
4810
4811         ifp->if_ipackets = s->rx_bcast_frames + s->rx_mcast_frames +
4812             s->rx_ucast_frames;
4813         ifp->if_ierrors = s->rx_err_frames;
4814         ifp->if_opackets = s->tx_bcast_frames + s->tx_mcast_frames +
4815             s->tx_ucast_frames + s->tx_offload_frames;
4816         ifp->if_oerrors = s->tx_drop_frames;
4817         ifp->if_ibytes = s->rx_bcast_bytes + s->rx_mcast_bytes +
4818             s->rx_ucast_bytes;
4819         ifp->if_obytes = s->tx_bcast_bytes + s->tx_mcast_bytes +
4820             s->tx_ucast_bytes + s->tx_offload_bytes;
4821         ifp->if_imcasts = s->rx_mcast_frames;
4822         ifp->if_omcasts = s->tx_mcast_frames;
4823
4824         drops = 0;
4825         for_each_txq(vi, i, txq)
4826                 drops += counter_u64_fetch(txq->r->drops);
4827         ifp->if_snd.ifq_drops = drops;
4828
4829         getmicrotime(&vi->last_refreshed);
4830         mtx_unlock(&sc->regwin_lock);
4831 }
4832
4833 static void
4834 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4835 {
4836         struct vi_info *vi = &pi->vi[0];
4837         struct ifnet *ifp = vi->ifp;
4838         struct sge_txq *txq;
4839         int i, drops;
4840         struct port_stats *s = &pi->stats;
4841         struct timeval tv;
4842         const struct timeval interval = {0, 250000};    /* 250ms */
4843
4844         getmicrotime(&tv);
4845         timevalsub(&tv, &interval);
4846         if (timevalcmp(&tv, &pi->last_refreshed, <))
4847                 return;
4848
4849         t4_get_port_stats(sc, pi->tx_chan, s);
4850
4851         ifp->if_opackets = s->tx_frames - s->tx_pause;
4852         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4853         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4854         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4855         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4856         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4857         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4858             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4859             s->rx_trunc3;
4860         for (i = 0; i < NCHAN; i++) {
4861                 if (pi->rx_chan_map & (1 << i)) {
4862                         uint32_t v;
4863
4864                         mtx_lock(&sc->regwin_lock);
4865                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4866                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4867                         mtx_unlock(&sc->regwin_lock);
4868                         ifp->if_iqdrops += v;
4869                 }
4870         }
4871
4872         drops = s->tx_drop;
4873         for_each_txq(vi, i, txq)
4874                 drops += counter_u64_fetch(txq->r->drops);
4875         ifp->if_snd.ifq_drops = drops;
4876
4877         ifp->if_oerrors = s->tx_error_frames;
4878         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4879             s->rx_fcs_err + s->rx_len_err;
4880
4881         getmicrotime(&pi->last_refreshed);
4882 }
4883
4884 static void
4885 cxgbe_tick(void *arg)
4886 {
4887         struct port_info *pi = arg;
4888         struct adapter *sc = pi->adapter;
4889
4890         PORT_LOCK_ASSERT_OWNED(pi);
4891         cxgbe_refresh_stats(sc, pi);
4892
4893         callout_schedule(&pi->tick, hz);
4894 }
4895
4896 void
4897 vi_tick(void *arg)
4898 {
4899         struct vi_info *vi = arg;
4900         struct adapter *sc = vi->pi->adapter;
4901
4902         vi_refresh_stats(sc, vi);
4903
4904         callout_schedule(&vi->tick, hz);
4905 }
4906
4907 static void
4908 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4909 {
4910         struct ifnet *vlan;
4911
4912         if (arg != ifp || ifp->if_type != IFT_ETHER)
4913                 return;
4914
4915         vlan = VLAN_DEVAT(ifp, vid);
4916         VLAN_SETCOOKIE(vlan, ifp);
4917 }
4918
4919 static int
4920 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4921 {
4922
4923 #ifdef INVARIANTS
4924         panic("%s: opcode 0x%02x on iq %p with payload %p",
4925             __func__, rss->opcode, iq, m);
4926 #else
4927         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4928             __func__, rss->opcode, iq, m);
4929         m_freem(m);
4930 #endif
4931         return (EDOOFUS);
4932 }
4933
4934 int
4935 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4936 {
4937         uintptr_t *loc, new;
4938
4939         if (opcode >= nitems(sc->cpl_handler))
4940                 return (EINVAL);
4941
4942         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4943         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4944         atomic_store_rel_ptr(loc, new);
4945
4946         return (0);
4947 }
4948
4949 static int
4950 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4951 {
4952
4953 #ifdef INVARIANTS
4954         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4955 #else
4956         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4957             __func__, iq, ctrl);
4958 #endif
4959         return (EDOOFUS);
4960 }
4961
4962 int
4963 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4964 {
4965         uintptr_t *loc, new;
4966
4967         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4968         loc = (uintptr_t *) &sc->an_handler;
4969         atomic_store_rel_ptr(loc, new);
4970
4971         return (0);
4972 }
4973
4974 static int
4975 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4976 {
4977         const struct cpl_fw6_msg *cpl =
4978             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4979
4980 #ifdef INVARIANTS
4981         panic("%s: fw_msg type %d", __func__, cpl->type);
4982 #else
4983         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4984 #endif
4985         return (EDOOFUS);
4986 }
4987
4988 int
4989 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4990 {
4991         uintptr_t *loc, new;
4992
4993         if (type >= nitems(sc->fw_msg_handler))
4994                 return (EINVAL);
4995
4996         /*
4997          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4998          * handler dispatch table.  Reject any attempt to install a handler for
4999          * this subtype.
5000          */
5001         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
5002                 return (EINVAL);
5003
5004         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
5005         loc = (uintptr_t *) &sc->fw_msg_handler[type];
5006         atomic_store_rel_ptr(loc, new);
5007
5008         return (0);
5009 }
5010
5011 static void
5012 t4_sysctls(struct adapter *sc)
5013 {
5014         struct sysctl_ctx_list *ctx;
5015         struct sysctl_oid *oid;
5016         struct sysctl_oid_list *children, *c0;
5017         static char *caps[] = {
5018                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
5019                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
5020                     "\6HASHFILTER\7ETHOFLD",
5021                 "\20\1TOE",                             /* caps[2] toecaps */
5022                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
5023                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
5024                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
5025                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
5026                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
5027                     "\4PO_INITIAOR\5PO_TARGET"
5028         };
5029         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
5030
5031         ctx = device_get_sysctl_ctx(sc->dev);
5032
5033         /*
5034          * dev.t4nex.X.
5035          */
5036         oid = device_get_sysctl_tree(sc->dev);
5037         c0 = children = SYSCTL_CHILDREN(oid);
5038
5039         sc->sc_do_rxcopy = 1;
5040         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
5041             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
5042
5043         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
5044             sc->params.nports, "# of ports");
5045
5046         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
5047             NULL, chip_rev(sc), "chip hardware revision");
5048
5049         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
5050             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
5051
5052         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
5053             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
5054
5055         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
5056             sc->cfcsum, "config file checksum");
5057
5058         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
5059             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
5060             sysctl_bitfield, "A", "available doorbells");
5061
5062         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
5063             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
5064             sysctl_bitfield, "A", "available link capabilities");
5065
5066         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
5067             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
5068             sysctl_bitfield, "A", "available NIC capabilities");
5069
5070         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
5071             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
5072             sysctl_bitfield, "A", "available TCP offload capabilities");
5073
5074         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
5075             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
5076             sysctl_bitfield, "A", "available RDMA capabilities");
5077
5078         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
5079             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
5080             sysctl_bitfield, "A", "available iSCSI capabilities");
5081
5082         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
5083             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
5084             sysctl_bitfield, "A", "available FCoE capabilities");
5085
5086         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
5087             sc->params.vpd.cclk, "core clock frequency (in KHz)");
5088
5089         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
5090             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
5091             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
5092             "interrupt holdoff timer values (us)");
5093
5094         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
5095             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
5096             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
5097             "interrupt holdoff packet counter values");
5098
5099         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
5100             NULL, sc->tids.nftids, "number of filters");
5101
5102         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
5103             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
5104             "chip temperature (in Celsius)");
5105
5106         t4_sge_sysctls(sc, ctx, children);
5107
5108         sc->lro_timeout = 100;
5109         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
5110             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
5111
5112         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
5113             &sc->debug_flags, 0, "flags to enable runtime debugging");
5114
5115 #ifdef SBUF_DRAIN
5116         /*
5117          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
5118          */
5119         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
5120             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
5121             "logs and miscellaneous information");
5122         children = SYSCTL_CHILDREN(oid);
5123
5124         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
5125             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5126             sysctl_cctrl, "A", "congestion control");
5127
5128         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
5129             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5130             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
5131
5132         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
5133             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
5134             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
5135
5136         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
5137             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
5138             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
5139
5140         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
5141             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
5142             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
5143
5144         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
5145             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
5146             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
5147
5148         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
5149             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
5150             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
5151
5152         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
5153             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5154             sysctl_cim_la, "A", "CIM logic analyzer");
5155
5156         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
5157             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5158             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
5159
5160         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
5161             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
5162             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
5163
5164         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
5165             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
5166             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
5167
5168         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
5169             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
5170             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
5171
5172         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
5173             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
5174             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
5175
5176         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
5177             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
5178             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
5179
5180         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
5181             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
5182             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
5183
5184         if (is_t5(sc)) {
5185                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
5186                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
5187                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
5188
5189                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
5190                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
5191                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
5192         }
5193
5194         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
5195             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5196             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
5197
5198         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
5199             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5200             sysctl_cim_qcfg, "A", "CIM queue configuration");
5201
5202         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5203             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5204             sysctl_cpl_stats, "A", "CPL statistics");
5205
5206         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5207             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5208             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5209
5210         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5211             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5212             sysctl_devlog, "A", "firmware's device log");
5213
5214         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5215             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5216             sysctl_fcoe_stats, "A", "FCoE statistics");
5217
5218         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5219             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5220             sysctl_hw_sched, "A", "hardware scheduler ");
5221
5222         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5223             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5224             sysctl_l2t, "A", "hardware L2 table");
5225
5226         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5227             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5228             sysctl_lb_stats, "A", "loopback statistics");
5229
5230         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5231             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5232             sysctl_meminfo, "A", "memory regions");
5233
5234         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5235             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5236             sysctl_mps_tcam, "A", "MPS TCAM entries");
5237
5238         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5239             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5240             sysctl_path_mtus, "A", "path MTUs");
5241
5242         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5243             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5244             sysctl_pm_stats, "A", "PM statistics");
5245
5246         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5247             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5248             sysctl_rdma_stats, "A", "RDMA statistics");
5249
5250         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5251             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5252             sysctl_tcp_stats, "A", "TCP statistics");
5253
5254         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5255             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5256             sysctl_tids, "A", "TID information");
5257
5258         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5259             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5260             sysctl_tp_err_stats, "A", "TP error statistics");
5261
5262         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5263             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5264             sysctl_tp_la, "A", "TP logic analyzer");
5265
5266         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5267             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5268             sysctl_tx_rate, "A", "Tx rate");
5269
5270         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5271             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5272             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5273
5274         if (is_t5(sc)) {
5275                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5276                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5277                     sysctl_wcwr_stats, "A", "write combined work requests");
5278         }
5279 #endif
5280
5281 #ifdef TCP_OFFLOAD
5282         if (is_offload(sc)) {
5283                 /*
5284                  * dev.t4nex.X.toe.
5285                  */
5286                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5287                     NULL, "TOE parameters");
5288                 children = SYSCTL_CHILDREN(oid);
5289
5290                 sc->tt.sndbuf = 256 * 1024;
5291                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5292                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
5293
5294                 sc->tt.ddp = 0;
5295                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5296                     &sc->tt.ddp, 0, "DDP allowed");
5297
5298                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
5299                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
5300                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
5301
5302                 sc->tt.ddp_thres =
5303                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
5304                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
5305                     &sc->tt.ddp_thres, 0, "DDP threshold");
5306
5307                 sc->tt.rx_coalesce = 1;
5308                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5309                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5310
5311                 sc->tt.tx_align = 1;
5312                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5313                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5314         }
5315 #endif
5316 }
5317
5318 void
5319 vi_sysctls(struct vi_info *vi)
5320 {
5321         struct sysctl_ctx_list *ctx;
5322         struct sysctl_oid *oid;
5323         struct sysctl_oid_list *children;
5324
5325         ctx = device_get_sysctl_ctx(vi->dev);
5326
5327         /*
5328          * dev.v?(cxgbe|cxl).X.
5329          */
5330         oid = device_get_sysctl_tree(vi->dev);
5331         children = SYSCTL_CHILDREN(oid);
5332
5333         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
5334             vi->viid, "VI identifer");
5335         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5336             &vi->nrxq, 0, "# of rx queues");
5337         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5338             &vi->ntxq, 0, "# of tx queues");
5339         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5340             &vi->first_rxq, 0, "index of first rx queue");
5341         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5342             &vi->first_txq, 0, "index of first tx queue");
5343
5344         if (IS_MAIN_VI(vi)) {
5345                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
5346                     CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
5347                     "Reserve queue 0 for non-flowid packets");
5348         }
5349
5350 #ifdef TCP_OFFLOAD
5351         if (vi->nofldrxq != 0) {
5352                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5353                     &vi->nofldrxq, 0,
5354                     "# of rx queues for offloaded TCP connections");
5355                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5356                     &vi->nofldtxq, 0,
5357                     "# of tx queues for offloaded TCP connections");
5358                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5359                     CTLFLAG_RD, &vi->first_ofld_rxq, 0,
5360                     "index of first TOE rx queue");
5361                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5362                     CTLFLAG_RD, &vi->first_ofld_txq, 0,
5363                     "index of first TOE tx queue");
5364         }
5365 #endif
5366 #ifdef DEV_NETMAP
5367         if (vi->nnmrxq != 0) {
5368                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
5369                     &vi->nnmrxq, 0, "# of netmap rx queues");
5370                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
5371                     &vi->nnmtxq, 0, "# of netmap tx queues");
5372                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
5373                     CTLFLAG_RD, &vi->first_nm_rxq, 0,
5374                     "index of first netmap rx queue");
5375                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
5376                     CTLFLAG_RD, &vi->first_nm_txq, 0,
5377                     "index of first netmap tx queue");
5378         }
5379 #endif
5380
5381         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5382             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
5383             "holdoff timer index");
5384         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5385             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
5386             "holdoff packet counter index");
5387
5388         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5389             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
5390             "rx queue size");
5391         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5392             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
5393             "tx queue size");
5394 }
5395
5396 static void
5397 cxgbe_sysctls(struct port_info *pi)
5398 {
5399         struct sysctl_ctx_list *ctx;
5400         struct sysctl_oid *oid;
5401         struct sysctl_oid_list *children;
5402         struct adapter *sc = pi->adapter;
5403
5404         ctx = device_get_sysctl_ctx(pi->dev);
5405
5406         /*
5407          * dev.cxgbe.X.
5408          */
5409         oid = device_get_sysctl_tree(pi->dev);
5410         children = SYSCTL_CHILDREN(oid);
5411
5412         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5413            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5414         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5415                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5416                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5417                     "PHY temperature (in Celsius)");
5418                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5419                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5420                     "PHY firmware version");
5421         }
5422
5423         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5424             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
5425             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5426
5427         /*
5428          * dev.cxgbe.X.stats.
5429          */
5430         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5431             NULL, "port statistics");
5432         children = SYSCTL_CHILDREN(oid);
5433         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5434             &pi->tx_parse_error, 0,
5435             "# of tx packets with invalid length or # of segments");
5436
5437 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5438         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5439             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5440             sysctl_handle_t4_reg64, "QU", desc)
5441
5442         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5443             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5444         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5445             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5446         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5447             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5448         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5449             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5450         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5451             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5452         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5453             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5454         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5455             "# of tx frames in this range",
5456             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5457         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5458             "# of tx frames in this range",
5459             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5460         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5461             "# of tx frames in this range",
5462             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5463         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5464             "# of tx frames in this range",
5465             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5466         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5467             "# of tx frames in this range",
5468             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5469         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5470             "# of tx frames in this range",
5471             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5472         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5473             "# of tx frames in this range",
5474             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5475         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5476             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5477         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5478             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5479         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5480             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5481         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5482             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5483         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5484             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5485         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5486             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5487         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5488             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5489         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5490             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5491         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5492             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5493         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5494             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5495
5496         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5497             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5498         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5499             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5500         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5501             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5502         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5503             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5504         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5505             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5506         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5507             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5508         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5509             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5510         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5511             "# of frames received with bad FCS",
5512             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5513         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5514             "# of frames received with length error",
5515             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5516         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5517             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5518         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5519             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5520         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5521             "# of rx frames in this range",
5522             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5523         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5524             "# of rx frames in this range",
5525             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5526         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5527             "# of rx frames in this range",
5528             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5529         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5530             "# of rx frames in this range",
5531             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5532         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5533             "# of rx frames in this range",
5534             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5535         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5536             "# of rx frames in this range",
5537             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5538         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5539             "# of rx frames in this range",
5540             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5541         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5542             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5543         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5544             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5545         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5546             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5547         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5548             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5549         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5550             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5551         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5552             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5553         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5554             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5555         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5556             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5557         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5558             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5559
5560 #undef SYSCTL_ADD_T4_REG64
5561
5562 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5563         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5564             &pi->stats.name, desc)
5565
5566         /* We get these from port_stats and they may be stale by upto 1s */
5567         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5568             "# drops due to buffer-group 0 overflows");
5569         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5570             "# drops due to buffer-group 1 overflows");
5571         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5572             "# drops due to buffer-group 2 overflows");
5573         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5574             "# drops due to buffer-group 3 overflows");
5575         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5576             "# of buffer-group 0 truncated packets");
5577         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5578             "# of buffer-group 1 truncated packets");
5579         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5580             "# of buffer-group 2 truncated packets");
5581         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5582             "# of buffer-group 3 truncated packets");
5583
5584 #undef SYSCTL_ADD_T4_PORTSTAT
5585 }
5586
5587 static int
5588 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5589 {
5590         int rc, *i, space = 0;
5591         struct sbuf sb;
5592
5593         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5594         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5595                 if (space)
5596                         sbuf_printf(&sb, " ");
5597                 sbuf_printf(&sb, "%d", *i);
5598                 space = 1;
5599         }
5600         sbuf_finish(&sb);
5601         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5602         sbuf_delete(&sb);
5603         return (rc);
5604 }
5605
5606 static int
5607 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5608 {
5609         int rc;
5610         struct sbuf *sb;
5611
5612         rc = sysctl_wire_old_buffer(req, 0);
5613         if (rc != 0)
5614                 return(rc);
5615
5616         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5617         if (sb == NULL)
5618                 return (ENOMEM);
5619
5620         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5621         rc = sbuf_finish(sb);
5622         sbuf_delete(sb);
5623
5624         return (rc);
5625 }
5626
5627 static int
5628 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5629 {
5630         struct port_info *pi = arg1;
5631         int op = arg2;
5632         struct adapter *sc = pi->adapter;
5633         u_int v;
5634         int rc;
5635
5636         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
5637         if (rc)
5638                 return (rc);
5639         /* XXX: magic numbers */
5640         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5641             &v);
5642         end_synchronized_op(sc, 0);
5643         if (rc)
5644                 return (rc);
5645         if (op == 0)
5646                 v /= 256;
5647
5648         rc = sysctl_handle_int(oidp, &v, 0, req);
5649         return (rc);
5650 }
5651
5652 static int
5653 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5654 {
5655         struct vi_info *vi = arg1;
5656         int rc, val;
5657
5658         val = vi->rsrv_noflowq;
5659         rc = sysctl_handle_int(oidp, &val, 0, req);
5660         if (rc != 0 || req->newptr == NULL)
5661                 return (rc);
5662
5663         if ((val >= 1) && (vi->ntxq > 1))
5664                 vi->rsrv_noflowq = 1;
5665         else
5666                 vi->rsrv_noflowq = 0;
5667
5668         return (rc);
5669 }
5670
5671 static int
5672 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5673 {
5674         struct vi_info *vi = arg1;
5675         struct adapter *sc = vi->pi->adapter;
5676         int idx, rc, i;
5677         struct sge_rxq *rxq;
5678 #ifdef TCP_OFFLOAD
5679         struct sge_ofld_rxq *ofld_rxq;
5680 #endif
5681         uint8_t v;
5682
5683         idx = vi->tmr_idx;
5684
5685         rc = sysctl_handle_int(oidp, &idx, 0, req);
5686         if (rc != 0 || req->newptr == NULL)
5687                 return (rc);
5688
5689         if (idx < 0 || idx >= SGE_NTIMERS)
5690                 return (EINVAL);
5691
5692         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5693             "t4tmr");
5694         if (rc)
5695                 return (rc);
5696
5697         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
5698         for_each_rxq(vi, i, rxq) {
5699 #ifdef atomic_store_rel_8
5700                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5701 #else
5702                 rxq->iq.intr_params = v;
5703 #endif
5704         }
5705 #ifdef TCP_OFFLOAD
5706         for_each_ofld_rxq(vi, i, ofld_rxq) {
5707 #ifdef atomic_store_rel_8
5708                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5709 #else
5710                 ofld_rxq->iq.intr_params = v;
5711 #endif
5712         }
5713 #endif
5714         vi->tmr_idx = idx;
5715
5716         end_synchronized_op(sc, LOCK_HELD);
5717         return (0);
5718 }
5719
5720 static int
5721 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5722 {
5723         struct vi_info *vi = arg1;
5724         struct adapter *sc = vi->pi->adapter;
5725         int idx, rc;
5726
5727         idx = vi->pktc_idx;
5728
5729         rc = sysctl_handle_int(oidp, &idx, 0, req);
5730         if (rc != 0 || req->newptr == NULL)
5731                 return (rc);
5732
5733         if (idx < -1 || idx >= SGE_NCOUNTERS)
5734                 return (EINVAL);
5735
5736         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5737             "t4pktc");
5738         if (rc)
5739                 return (rc);
5740
5741         if (vi->flags & VI_INIT_DONE)
5742                 rc = EBUSY; /* cannot be changed once the queues are created */
5743         else
5744                 vi->pktc_idx = idx;
5745
5746         end_synchronized_op(sc, LOCK_HELD);
5747         return (rc);
5748 }
5749
5750 static int
5751 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5752 {
5753         struct vi_info *vi = arg1;
5754         struct adapter *sc = vi->pi->adapter;
5755         int qsize, rc;
5756
5757         qsize = vi->qsize_rxq;
5758
5759         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5760         if (rc != 0 || req->newptr == NULL)
5761                 return (rc);
5762
5763         if (qsize < 128 || (qsize & 7))
5764                 return (EINVAL);
5765
5766         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5767             "t4rxqs");
5768         if (rc)
5769                 return (rc);
5770
5771         if (vi->flags & VI_INIT_DONE)
5772                 rc = EBUSY; /* cannot be changed once the queues are created */
5773         else
5774                 vi->qsize_rxq = qsize;
5775
5776         end_synchronized_op(sc, LOCK_HELD);
5777         return (rc);
5778 }
5779
5780 static int
5781 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5782 {
5783         struct vi_info *vi = arg1;
5784         struct adapter *sc = vi->pi->adapter;
5785         int qsize, rc;
5786
5787         qsize = vi->qsize_txq;
5788
5789         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5790         if (rc != 0 || req->newptr == NULL)
5791                 return (rc);
5792
5793         if (qsize < 128 || qsize > 65536)
5794                 return (EINVAL);
5795
5796         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5797             "t4txqs");
5798         if (rc)
5799                 return (rc);
5800
5801         if (vi->flags & VI_INIT_DONE)
5802                 rc = EBUSY; /* cannot be changed once the queues are created */
5803         else
5804                 vi->qsize_txq = qsize;
5805
5806         end_synchronized_op(sc, LOCK_HELD);
5807         return (rc);
5808 }
5809
5810 static int
5811 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5812 {
5813         struct port_info *pi = arg1;
5814         struct adapter *sc = pi->adapter;
5815         struct link_config *lc = &pi->link_cfg;
5816         int rc;
5817
5818         if (req->newptr == NULL) {
5819                 struct sbuf *sb;
5820                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5821
5822                 rc = sysctl_wire_old_buffer(req, 0);
5823                 if (rc != 0)
5824                         return(rc);
5825
5826                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5827                 if (sb == NULL)
5828                         return (ENOMEM);
5829
5830                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5831                 rc = sbuf_finish(sb);
5832                 sbuf_delete(sb);
5833         } else {
5834                 char s[2];
5835                 int n;
5836
5837                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5838                 s[1] = 0;
5839
5840                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5841                 if (rc != 0)
5842                         return(rc);
5843
5844                 if (s[1] != 0)
5845                         return (EINVAL);
5846                 if (s[0] < '0' || s[0] > '9')
5847                         return (EINVAL);        /* not a number */
5848                 n = s[0] - '0';
5849                 if (n & ~(PAUSE_TX | PAUSE_RX))
5850                         return (EINVAL);        /* some other bit is set too */
5851
5852                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5853                     "t4PAUSE");
5854                 if (rc)
5855                         return (rc);
5856                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5857                         int link_ok = lc->link_ok;
5858
5859                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5860                         lc->requested_fc |= n;
5861                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5862                         lc->link_ok = link_ok;  /* restore */
5863                 }
5864                 end_synchronized_op(sc, 0);
5865         }
5866
5867         return (rc);
5868 }
5869
5870 static int
5871 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5872 {
5873         struct adapter *sc = arg1;
5874         int reg = arg2;
5875         uint64_t val;
5876
5877         val = t4_read_reg64(sc, reg);
5878
5879         return (sysctl_handle_64(oidp, &val, 0, req));
5880 }
5881
5882 static int
5883 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5884 {
5885         struct adapter *sc = arg1;
5886         int rc, t;
5887         uint32_t param, val;
5888
5889         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5890         if (rc)
5891                 return (rc);
5892         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5893             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5894             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5895         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5896         end_synchronized_op(sc, 0);
5897         if (rc)
5898                 return (rc);
5899
5900         /* unknown is returned as 0 but we display -1 in that case */
5901         t = val == 0 ? -1 : val;
5902
5903         rc = sysctl_handle_int(oidp, &t, 0, req);
5904         return (rc);
5905 }
5906
5907 #ifdef SBUF_DRAIN
5908 static int
5909 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5910 {
5911         struct adapter *sc = arg1;
5912         struct sbuf *sb;
5913         int rc, i;
5914         uint16_t incr[NMTUS][NCCTRL_WIN];
5915         static const char *dec_fac[] = {
5916                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5917                 "0.9375"
5918         };
5919
5920         rc = sysctl_wire_old_buffer(req, 0);
5921         if (rc != 0)
5922                 return (rc);
5923
5924         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5925         if (sb == NULL)
5926                 return (ENOMEM);
5927
5928         t4_read_cong_tbl(sc, incr);
5929
5930         for (i = 0; i < NCCTRL_WIN; ++i) {
5931                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5932                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5933                     incr[5][i], incr[6][i], incr[7][i]);
5934                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5935                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5936                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5937                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5938         }
5939
5940         rc = sbuf_finish(sb);
5941         sbuf_delete(sb);
5942
5943         return (rc);
5944 }
5945
5946 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5947         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5948         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5949         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5950 };
5951
5952 static int
5953 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5954 {
5955         struct adapter *sc = arg1;
5956         struct sbuf *sb;
5957         int rc, i, n, qid = arg2;
5958         uint32_t *buf, *p;
5959         char *qtype;
5960         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5961
5962         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5963             ("%s: bad qid %d\n", __func__, qid));
5964
5965         if (qid < CIM_NUM_IBQ) {
5966                 /* inbound queue */
5967                 qtype = "IBQ";
5968                 n = 4 * CIM_IBQ_SIZE;
5969                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5970                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5971         } else {
5972                 /* outbound queue */
5973                 qtype = "OBQ";
5974                 qid -= CIM_NUM_IBQ;
5975                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5976                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5977                 rc = t4_read_cim_obq(sc, qid, buf, n);
5978         }
5979
5980         if (rc < 0) {
5981                 rc = -rc;
5982                 goto done;
5983         }
5984         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5985
5986         rc = sysctl_wire_old_buffer(req, 0);
5987         if (rc != 0)
5988                 goto done;
5989
5990         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5991         if (sb == NULL) {
5992                 rc = ENOMEM;
5993                 goto done;
5994         }
5995
5996         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5997         for (i = 0, p = buf; i < n; i += 16, p += 4)
5998                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5999                     p[2], p[3]);
6000
6001         rc = sbuf_finish(sb);
6002         sbuf_delete(sb);
6003 done:
6004         free(buf, M_CXGBE);
6005         return (rc);
6006 }
6007
6008 static int
6009 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
6010 {
6011         struct adapter *sc = arg1;
6012         u_int cfg;
6013         struct sbuf *sb;
6014         uint32_t *buf, *p;
6015         int rc;
6016
6017         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
6018         if (rc != 0)
6019                 return (rc);
6020
6021         rc = sysctl_wire_old_buffer(req, 0);
6022         if (rc != 0)
6023                 return (rc);
6024
6025         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6026         if (sb == NULL)
6027                 return (ENOMEM);
6028
6029         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
6030             M_ZERO | M_WAITOK);
6031
6032         rc = -t4_cim_read_la(sc, buf, NULL);
6033         if (rc != 0)
6034                 goto done;
6035
6036         sbuf_printf(sb, "Status   Data      PC%s",
6037             cfg & F_UPDBGLACAPTPCONLY ? "" :
6038             "     LS0Stat  LS0Addr             LS0Data");
6039
6040         KASSERT((sc->params.cim_la_size & 7) == 0,
6041             ("%s: p will walk off the end of buf", __func__));
6042
6043         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
6044                 if (cfg & F_UPDBGLACAPTPCONLY) {
6045                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
6046                             p[6], p[7]);
6047                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
6048                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
6049                             p[4] & 0xff, p[5] >> 8);
6050                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
6051                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6052                             p[1] & 0xf, p[2] >> 4);
6053                 } else {
6054                         sbuf_printf(sb,
6055                             "\n  %02x   %x%07x %x%07x %08x %08x "
6056                             "%08x%08x%08x%08x",
6057                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6058                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
6059                             p[6], p[7]);
6060                 }
6061         }
6062
6063         rc = sbuf_finish(sb);
6064         sbuf_delete(sb);
6065 done:
6066         free(buf, M_CXGBE);
6067         return (rc);
6068 }
6069
6070 static int
6071 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
6072 {
6073         struct adapter *sc = arg1;
6074         u_int i;
6075         struct sbuf *sb;
6076         uint32_t *buf, *p;
6077         int rc;
6078
6079         rc = sysctl_wire_old_buffer(req, 0);
6080         if (rc != 0)
6081                 return (rc);
6082
6083         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6084         if (sb == NULL)
6085                 return (ENOMEM);
6086
6087         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
6088             M_ZERO | M_WAITOK);
6089
6090         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
6091         p = buf;
6092
6093         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6094                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
6095                     p[1], p[0]);
6096         }
6097
6098         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
6099         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6100                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
6101                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
6102                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
6103                     (p[1] >> 2) | ((p[2] & 3) << 30),
6104                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
6105                     p[0] & 1);
6106         }
6107
6108         rc = sbuf_finish(sb);
6109         sbuf_delete(sb);
6110         free(buf, M_CXGBE);
6111         return (rc);
6112 }
6113
6114 static int
6115 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
6116 {
6117         struct adapter *sc = arg1;
6118         u_int i;
6119         struct sbuf *sb;
6120         uint32_t *buf, *p;
6121         int rc;
6122
6123         rc = sysctl_wire_old_buffer(req, 0);
6124         if (rc != 0)
6125                 return (rc);
6126
6127         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6128         if (sb == NULL)
6129                 return (ENOMEM);
6130
6131         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
6132             M_ZERO | M_WAITOK);
6133
6134         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
6135         p = buf;
6136
6137         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
6138         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6139                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
6140                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
6141                     p[4], p[3], p[2], p[1], p[0]);
6142         }
6143
6144         sbuf_printf(sb, "\n\nCntl ID               Data");
6145         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6146                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
6147                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
6148         }
6149
6150         rc = sbuf_finish(sb);
6151         sbuf_delete(sb);
6152         free(buf, M_CXGBE);
6153         return (rc);
6154 }
6155
6156 static int
6157 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
6158 {
6159         struct adapter *sc = arg1;
6160         struct sbuf *sb;
6161         int rc, i;
6162         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6163         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6164         uint16_t thres[CIM_NUM_IBQ];
6165         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
6166         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
6167         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
6168
6169         if (is_t4(sc)) {
6170                 cim_num_obq = CIM_NUM_OBQ;
6171                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
6172                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
6173         } else {
6174                 cim_num_obq = CIM_NUM_OBQ_T5;
6175                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
6176                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
6177         }
6178         nq = CIM_NUM_IBQ + cim_num_obq;
6179
6180         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
6181         if (rc == 0)
6182                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
6183         if (rc != 0)
6184                 return (rc);
6185
6186         t4_read_cimq_cfg(sc, base, size, thres);
6187
6188         rc = sysctl_wire_old_buffer(req, 0);
6189         if (rc != 0)
6190                 return (rc);
6191
6192         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6193         if (sb == NULL)
6194                 return (ENOMEM);
6195
6196         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
6197
6198         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
6199                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
6200                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
6201                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6202                     G_QUEREMFLITS(p[2]) * 16);
6203         for ( ; i < nq; i++, p += 4, wr += 2)
6204                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
6205                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
6206                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6207                     G_QUEREMFLITS(p[2]) * 16);
6208
6209         rc = sbuf_finish(sb);
6210         sbuf_delete(sb);
6211
6212         return (rc);
6213 }
6214
6215 static int
6216 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
6217 {
6218         struct adapter *sc = arg1;
6219         struct sbuf *sb;
6220         int rc;
6221         struct tp_cpl_stats stats;
6222
6223         rc = sysctl_wire_old_buffer(req, 0);
6224         if (rc != 0)
6225                 return (rc);
6226
6227         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6228         if (sb == NULL)
6229                 return (ENOMEM);
6230
6231         t4_tp_get_cpl_stats(sc, &stats);
6232
6233         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6234             "channel 3\n");
6235         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
6236                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
6237         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
6238                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
6239
6240         rc = sbuf_finish(sb);
6241         sbuf_delete(sb);
6242
6243         return (rc);
6244 }
6245
6246 static int
6247 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
6248 {
6249         struct adapter *sc = arg1;
6250         struct sbuf *sb;
6251         int rc;
6252         struct tp_usm_stats stats;
6253
6254         rc = sysctl_wire_old_buffer(req, 0);
6255         if (rc != 0)
6256                 return(rc);
6257
6258         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6259         if (sb == NULL)
6260                 return (ENOMEM);
6261
6262         t4_get_usm_stats(sc, &stats);
6263
6264         sbuf_printf(sb, "Frames: %u\n", stats.frames);
6265         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
6266         sbuf_printf(sb, "Drops:  %u", stats.drops);
6267
6268         rc = sbuf_finish(sb);
6269         sbuf_delete(sb);
6270
6271         return (rc);
6272 }
6273
6274 const char *devlog_level_strings[] = {
6275         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
6276         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
6277         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
6278         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
6279         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
6280         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
6281 };
6282
6283 const char *devlog_facility_strings[] = {
6284         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
6285         [FW_DEVLOG_FACILITY_CF]         = "CF",
6286         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
6287         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
6288         [FW_DEVLOG_FACILITY_RES]        = "RES",
6289         [FW_DEVLOG_FACILITY_HW]         = "HW",
6290         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
6291         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
6292         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
6293         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
6294         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
6295         [FW_DEVLOG_FACILITY_VI]         = "VI",
6296         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
6297         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
6298         [FW_DEVLOG_FACILITY_TM]         = "TM",
6299         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
6300         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
6301         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
6302         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
6303         [FW_DEVLOG_FACILITY_RI]         = "RI",
6304         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
6305         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
6306         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
6307         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
6308 };
6309
6310 static int
6311 sysctl_devlog(SYSCTL_HANDLER_ARGS)
6312 {
6313         struct adapter *sc = arg1;
6314         struct devlog_params *dparams = &sc->params.devlog;
6315         struct fw_devlog_e *buf, *e;
6316         int i, j, rc, nentries, first = 0, m;
6317         struct sbuf *sb;
6318         uint64_t ftstamp = UINT64_MAX;
6319
6320         if (dparams->start == 0) {
6321                 dparams->memtype = FW_MEMTYPE_EDC0;
6322                 dparams->start = 0x84000;
6323                 dparams->size = 32768;
6324         }
6325
6326         nentries = dparams->size / sizeof(struct fw_devlog_e);
6327
6328         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
6329         if (buf == NULL)
6330                 return (ENOMEM);
6331
6332         m = fwmtype_to_hwmtype(dparams->memtype);
6333         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
6334         if (rc != 0)
6335                 goto done;
6336
6337         for (i = 0; i < nentries; i++) {
6338                 e = &buf[i];
6339
6340                 if (e->timestamp == 0)
6341                         break;  /* end */
6342
6343                 e->timestamp = be64toh(e->timestamp);
6344                 e->seqno = be32toh(e->seqno);
6345                 for (j = 0; j < 8; j++)
6346                         e->params[j] = be32toh(e->params[j]);
6347
6348                 if (e->timestamp < ftstamp) {
6349                         ftstamp = e->timestamp;
6350                         first = i;
6351                 }
6352         }
6353
6354         if (buf[first].timestamp == 0)
6355                 goto done;      /* nothing in the log */
6356
6357         rc = sysctl_wire_old_buffer(req, 0);
6358         if (rc != 0)
6359                 goto done;
6360
6361         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6362         if (sb == NULL) {
6363                 rc = ENOMEM;
6364                 goto done;
6365         }
6366         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6367             "Seq#", "Tstamp", "Level", "Facility", "Message");
6368
6369         i = first;
6370         do {
6371                 e = &buf[i];
6372                 if (e->timestamp == 0)
6373                         break;  /* end */
6374
6375                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6376                     e->seqno, e->timestamp,
6377                     (e->level < nitems(devlog_level_strings) ?
6378                         devlog_level_strings[e->level] : "UNKNOWN"),
6379                     (e->facility < nitems(devlog_facility_strings) ?
6380                         devlog_facility_strings[e->facility] : "UNKNOWN"));
6381                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6382                     e->params[2], e->params[3], e->params[4],
6383                     e->params[5], e->params[6], e->params[7]);
6384
6385                 if (++i == nentries)
6386                         i = 0;
6387         } while (i != first);
6388
6389         rc = sbuf_finish(sb);
6390         sbuf_delete(sb);
6391 done:
6392         free(buf, M_CXGBE);
6393         return (rc);
6394 }
6395
6396 static int
6397 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6398 {
6399         struct adapter *sc = arg1;
6400         struct sbuf *sb;
6401         int rc;
6402         struct tp_fcoe_stats stats[4];
6403
6404         rc = sysctl_wire_old_buffer(req, 0);
6405         if (rc != 0)
6406                 return (rc);
6407
6408         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6409         if (sb == NULL)
6410                 return (ENOMEM);
6411
6412         t4_get_fcoe_stats(sc, 0, &stats[0]);
6413         t4_get_fcoe_stats(sc, 1, &stats[1]);
6414         t4_get_fcoe_stats(sc, 2, &stats[2]);
6415         t4_get_fcoe_stats(sc, 3, &stats[3]);
6416
6417         sbuf_printf(sb, "                   channel 0        channel 1        "
6418             "channel 2        channel 3\n");
6419         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
6420             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
6421             stats[3].octetsDDP);
6422         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
6423             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
6424         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
6425             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
6426             stats[3].framesDrop);
6427
6428         rc = sbuf_finish(sb);
6429         sbuf_delete(sb);
6430
6431         return (rc);
6432 }
6433
6434 static int
6435 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6436 {
6437         struct adapter *sc = arg1;
6438         struct sbuf *sb;
6439         int rc, i;
6440         unsigned int map, kbps, ipg, mode;
6441         unsigned int pace_tab[NTX_SCHED];
6442
6443         rc = sysctl_wire_old_buffer(req, 0);
6444         if (rc != 0)
6445                 return (rc);
6446
6447         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6448         if (sb == NULL)
6449                 return (ENOMEM);
6450
6451         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6452         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6453         t4_read_pace_tbl(sc, pace_tab);
6454
6455         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6456             "Class IPG (0.1 ns)   Flow IPG (us)");
6457
6458         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6459                 t4_get_tx_sched(sc, i, &kbps, &ipg);
6460                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6461                     (mode & (1 << i)) ? "flow" : "class", map & 3);
6462                 if (kbps)
6463                         sbuf_printf(sb, "%9u     ", kbps);
6464                 else
6465                         sbuf_printf(sb, " disabled     ");
6466
6467                 if (ipg)
6468                         sbuf_printf(sb, "%13u        ", ipg);
6469                 else
6470                         sbuf_printf(sb, "     disabled        ");
6471
6472                 if (pace_tab[i])
6473                         sbuf_printf(sb, "%10u", pace_tab[i]);
6474                 else
6475                         sbuf_printf(sb, "  disabled");
6476         }
6477
6478         rc = sbuf_finish(sb);
6479         sbuf_delete(sb);
6480
6481         return (rc);
6482 }
6483
6484 static int
6485 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6486 {
6487         struct adapter *sc = arg1;
6488         struct sbuf *sb;
6489         int rc, i, j;
6490         uint64_t *p0, *p1;
6491         struct lb_port_stats s[2];
6492         static const char *stat_name[] = {
6493                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6494                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6495                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
6496                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6497                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6498                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6499                 "BG2FramesTrunc:", "BG3FramesTrunc:"
6500         };
6501
6502         rc = sysctl_wire_old_buffer(req, 0);
6503         if (rc != 0)
6504                 return (rc);
6505
6506         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6507         if (sb == NULL)
6508                 return (ENOMEM);
6509
6510         memset(s, 0, sizeof(s));
6511
6512         for (i = 0; i < 4; i += 2) {
6513                 t4_get_lb_stats(sc, i, &s[0]);
6514                 t4_get_lb_stats(sc, i + 1, &s[1]);
6515
6516                 p0 = &s[0].octets;
6517                 p1 = &s[1].octets;
6518                 sbuf_printf(sb, "%s                       Loopback %u"
6519                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6520
6521                 for (j = 0; j < nitems(stat_name); j++)
6522                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6523                                    *p0++, *p1++);
6524         }
6525
6526         rc = sbuf_finish(sb);
6527         sbuf_delete(sb);
6528
6529         return (rc);
6530 }
6531
6532 static int
6533 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6534 {
6535         int rc = 0;
6536         struct port_info *pi = arg1;
6537         struct sbuf *sb;
6538         static const char *linkdnreasons[] = {
6539                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6540                 "PHY overheated", "unknown", "rx los", "reserved7"
6541         };
6542
6543         rc = sysctl_wire_old_buffer(req, 0);
6544         if (rc != 0)
6545                 return(rc);
6546         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6547         if (sb == NULL)
6548                 return (ENOMEM);
6549
6550         if (pi->linkdnrc < 0)
6551                 sbuf_printf(sb, "n/a");
6552         else if (pi->linkdnrc < nitems(linkdnreasons))
6553                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6554         else
6555                 sbuf_printf(sb, "%d", pi->linkdnrc);
6556
6557         rc = sbuf_finish(sb);
6558         sbuf_delete(sb);
6559
6560         return (rc);
6561 }
6562
6563 struct mem_desc {
6564         unsigned int base;
6565         unsigned int limit;
6566         unsigned int idx;
6567 };
6568
6569 static int
6570 mem_desc_cmp(const void *a, const void *b)
6571 {
6572         return ((const struct mem_desc *)a)->base -
6573                ((const struct mem_desc *)b)->base;
6574 }
6575
6576 static void
6577 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6578     unsigned int to)
6579 {
6580         unsigned int size;
6581
6582         size = to - from + 1;
6583         if (size == 0)
6584                 return;
6585
6586         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6587         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6588 }
6589
6590 static int
6591 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6592 {
6593         struct adapter *sc = arg1;
6594         struct sbuf *sb;
6595         int rc, i, n;
6596         uint32_t lo, hi, used, alloc;
6597         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6598         static const char *region[] = {
6599                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6600                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6601                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6602                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6603                 "RQUDP region:", "PBL region:", "TXPBL region:",
6604                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6605                 "On-chip queues:"
6606         };
6607         struct mem_desc avail[4];
6608         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6609         struct mem_desc *md = mem;
6610
6611         rc = sysctl_wire_old_buffer(req, 0);
6612         if (rc != 0)
6613                 return (rc);
6614
6615         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6616         if (sb == NULL)
6617                 return (ENOMEM);
6618
6619         for (i = 0; i < nitems(mem); i++) {
6620                 mem[i].limit = 0;
6621                 mem[i].idx = i;
6622         }
6623
6624         /* Find and sort the populated memory ranges */
6625         i = 0;
6626         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6627         if (lo & F_EDRAM0_ENABLE) {
6628                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6629                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6630                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6631                 avail[i].idx = 0;
6632                 i++;
6633         }
6634         if (lo & F_EDRAM1_ENABLE) {
6635                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6636                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6637                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6638                 avail[i].idx = 1;
6639                 i++;
6640         }
6641         if (lo & F_EXT_MEM_ENABLE) {
6642                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6643                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6644                 avail[i].limit = avail[i].base +
6645                     (G_EXT_MEM_SIZE(hi) << 20);
6646                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6647                 i++;
6648         }
6649         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6650                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6651                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6652                 avail[i].limit = avail[i].base +
6653                     (G_EXT_MEM1_SIZE(hi) << 20);
6654                 avail[i].idx = 4;
6655                 i++;
6656         }
6657         if (!i)                                    /* no memory available */
6658                 return 0;
6659         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6660
6661         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6662         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6663         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6664         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6665         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6666         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6667         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6668         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6669         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6670
6671         /* the next few have explicit upper bounds */
6672         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6673         md->limit = md->base - 1 +
6674                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6675                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6676         md++;
6677
6678         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6679         md->limit = md->base - 1 +
6680                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6681                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6682         md++;
6683
6684         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6685                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6686                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6687                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6688         } else {
6689                 md->base = 0;
6690                 md->idx = nitems(region);  /* hide it */
6691         }
6692         md++;
6693
6694 #define ulp_region(reg) \
6695         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6696         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6697
6698         ulp_region(RX_ISCSI);
6699         ulp_region(RX_TDDP);
6700         ulp_region(TX_TPT);
6701         ulp_region(RX_STAG);
6702         ulp_region(RX_RQ);
6703         ulp_region(RX_RQUDP);
6704         ulp_region(RX_PBL);
6705         ulp_region(TX_PBL);
6706 #undef ulp_region
6707
6708         md->base = 0;
6709         md->idx = nitems(region);
6710         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6711                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6712                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6713                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6714         }
6715         md++;
6716
6717         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6718         md->limit = md->base + sc->tids.ntids - 1;
6719         md++;
6720         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6721         md->limit = md->base + sc->tids.ntids - 1;
6722         md++;
6723
6724         md->base = sc->vres.ocq.start;
6725         if (sc->vres.ocq.size)
6726                 md->limit = md->base + sc->vres.ocq.size - 1;
6727         else
6728                 md->idx = nitems(region);  /* hide it */
6729         md++;
6730
6731         /* add any address-space holes, there can be up to 3 */
6732         for (n = 0; n < i - 1; n++)
6733                 if (avail[n].limit < avail[n + 1].base)
6734                         (md++)->base = avail[n].limit;
6735         if (avail[n].limit)
6736                 (md++)->base = avail[n].limit;
6737
6738         n = md - mem;
6739         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6740
6741         for (lo = 0; lo < i; lo++)
6742                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6743                                 avail[lo].limit - 1);
6744
6745         sbuf_printf(sb, "\n");
6746         for (i = 0; i < n; i++) {
6747                 if (mem[i].idx >= nitems(region))
6748                         continue;                        /* skip holes */
6749                 if (!mem[i].limit)
6750                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6751                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6752                                 mem[i].limit);
6753         }
6754
6755         sbuf_printf(sb, "\n");
6756         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6757         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6758         mem_region_show(sb, "uP RAM:", lo, hi);
6759
6760         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6761         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6762         mem_region_show(sb, "uP Extmem2:", lo, hi);
6763
6764         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6765         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6766                    G_PMRXMAXPAGE(lo),
6767                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6768                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6769
6770         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6771         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6772         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6773                    G_PMTXMAXPAGE(lo),
6774                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6775                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6776         sbuf_printf(sb, "%u p-structs\n",
6777                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6778
6779         for (i = 0; i < 4; i++) {
6780                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6781                 if (is_t4(sc)) {
6782                         used = G_USED(lo);
6783                         alloc = G_ALLOC(lo);
6784                 } else {
6785                         used = G_T5_USED(lo);
6786                         alloc = G_T5_ALLOC(lo);
6787                 }
6788                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6789                            i, used, alloc);
6790         }
6791         for (i = 0; i < 4; i++) {
6792                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6793                 if (is_t4(sc)) {
6794                         used = G_USED(lo);
6795                         alloc = G_ALLOC(lo);
6796                 } else {
6797                         used = G_T5_USED(lo);
6798                         alloc = G_T5_ALLOC(lo);
6799                 }
6800                 sbuf_printf(sb,
6801                            "\nLoopback %d using %u pages out of %u allocated",
6802                            i, used, alloc);
6803         }
6804
6805         rc = sbuf_finish(sb);
6806         sbuf_delete(sb);
6807
6808         return (rc);
6809 }
6810
6811 static inline void
6812 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6813 {
6814         *mask = x | y;
6815         y = htobe64(y);
6816         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6817 }
6818
6819 static int
6820 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6821 {
6822         struct adapter *sc = arg1;
6823         struct sbuf *sb;
6824         int rc, i, n;
6825
6826         rc = sysctl_wire_old_buffer(req, 0);
6827         if (rc != 0)
6828                 return (rc);
6829
6830         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6831         if (sb == NULL)
6832                 return (ENOMEM);
6833
6834         sbuf_printf(sb,
6835             "Idx  Ethernet address     Mask     Vld Ports PF"
6836             "  VF              Replication             P0 P1 P2 P3  ML");
6837         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6838             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6839         for (i = 0; i < n; i++) {
6840                 uint64_t tcamx, tcamy, mask;
6841                 uint32_t cls_lo, cls_hi;
6842                 uint8_t addr[ETHER_ADDR_LEN];
6843
6844                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6845                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6846                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6847                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6848
6849                 if (tcamx & tcamy)
6850                         continue;
6851
6852                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6853                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6854                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6855                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6856                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6857                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6858                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6859
6860                 if (cls_lo & F_REPLICATE) {
6861                         struct fw_ldst_cmd ldst_cmd;
6862
6863                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6864                         ldst_cmd.op_to_addrspace =
6865                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6866                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6867                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6868                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6869                         ldst_cmd.u.mps.rplc.fid_idx =
6870                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6871                                 V_FW_LDST_CMD_IDX(i));
6872
6873                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6874                             "t4mps");
6875                         if (rc)
6876                                 break;
6877                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6878                             sizeof(ldst_cmd), &ldst_cmd);
6879                         end_synchronized_op(sc, 0);
6880
6881                         if (rc != 0) {
6882                                 sbuf_printf(sb,
6883                                     " ------------ error %3u ------------", rc);
6884                                 rc = 0;
6885                         } else {
6886                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6887                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6888                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6889                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6890                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6891                         }
6892                 } else
6893                         sbuf_printf(sb, "%36s", "");
6894
6895                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6896                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6897                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6898         }
6899
6900         if (rc)
6901                 (void) sbuf_finish(sb);
6902         else
6903                 rc = sbuf_finish(sb);
6904         sbuf_delete(sb);
6905
6906         return (rc);
6907 }
6908
6909 static int
6910 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6911 {
6912         struct adapter *sc = arg1;
6913         struct sbuf *sb;
6914         int rc;
6915         uint16_t mtus[NMTUS];
6916
6917         rc = sysctl_wire_old_buffer(req, 0);
6918         if (rc != 0)
6919                 return (rc);
6920
6921         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6922         if (sb == NULL)
6923                 return (ENOMEM);
6924
6925         t4_read_mtu_tbl(sc, mtus, NULL);
6926
6927         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6928             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6929             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6930             mtus[14], mtus[15]);
6931
6932         rc = sbuf_finish(sb);
6933         sbuf_delete(sb);
6934
6935         return (rc);
6936 }
6937
6938 static int
6939 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6940 {
6941         struct adapter *sc = arg1;
6942         struct sbuf *sb;
6943         int rc, i;
6944         uint32_t cnt[PM_NSTATS];
6945         uint64_t cyc[PM_NSTATS];
6946         static const char *rx_stats[] = {
6947                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6948         };
6949         static const char *tx_stats[] = {
6950                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6951         };
6952
6953         rc = sysctl_wire_old_buffer(req, 0);
6954         if (rc != 0)
6955                 return (rc);
6956
6957         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6958         if (sb == NULL)
6959                 return (ENOMEM);
6960
6961         t4_pmtx_get_stats(sc, cnt, cyc);
6962         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6963         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6964                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6965                     cyc[i]);
6966
6967         t4_pmrx_get_stats(sc, cnt, cyc);
6968         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6969         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6970                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6971                     cyc[i]);
6972
6973         rc = sbuf_finish(sb);
6974         sbuf_delete(sb);
6975
6976         return (rc);
6977 }
6978
6979 static int
6980 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6981 {
6982         struct adapter *sc = arg1;
6983         struct sbuf *sb;
6984         int rc;
6985         struct tp_rdma_stats stats;
6986
6987         rc = sysctl_wire_old_buffer(req, 0);
6988         if (rc != 0)
6989                 return (rc);
6990
6991         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6992         if (sb == NULL)
6993                 return (ENOMEM);
6994
6995         t4_tp_get_rdma_stats(sc, &stats);
6996         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6997         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6998
6999         rc = sbuf_finish(sb);
7000         sbuf_delete(sb);
7001
7002         return (rc);
7003 }
7004
7005 static int
7006 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
7007 {
7008         struct adapter *sc = arg1;
7009         struct sbuf *sb;
7010         int rc;
7011         struct tp_tcp_stats v4, v6;
7012
7013         rc = sysctl_wire_old_buffer(req, 0);
7014         if (rc != 0)
7015                 return (rc);
7016
7017         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7018         if (sb == NULL)
7019                 return (ENOMEM);
7020
7021         t4_tp_get_tcp_stats(sc, &v4, &v6);
7022         sbuf_printf(sb,
7023             "                                IP                 IPv6\n");
7024         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
7025             v4.tcpOutRsts, v6.tcpOutRsts);
7026         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
7027             v4.tcpInSegs, v6.tcpInSegs);
7028         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
7029             v4.tcpOutSegs, v6.tcpOutSegs);
7030         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
7031             v4.tcpRetransSegs, v6.tcpRetransSegs);
7032
7033         rc = sbuf_finish(sb);
7034         sbuf_delete(sb);
7035
7036         return (rc);
7037 }
7038
7039 static int
7040 sysctl_tids(SYSCTL_HANDLER_ARGS)
7041 {
7042         struct adapter *sc = arg1;
7043         struct sbuf *sb;
7044         int rc;
7045         struct tid_info *t = &sc->tids;
7046
7047         rc = sysctl_wire_old_buffer(req, 0);
7048         if (rc != 0)
7049                 return (rc);
7050
7051         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7052         if (sb == NULL)
7053                 return (ENOMEM);
7054
7055         if (t->natids) {
7056                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
7057                     t->atids_in_use);
7058         }
7059
7060         if (t->ntids) {
7061                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7062                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
7063
7064                         if (b) {
7065                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
7066                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7067                                     t->ntids - 1);
7068                         } else {
7069                                 sbuf_printf(sb, "TID range: %u-%u",
7070                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7071                                     t->ntids - 1);
7072                         }
7073                 } else
7074                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
7075                 sbuf_printf(sb, ", in use: %u\n",
7076                     atomic_load_acq_int(&t->tids_in_use));
7077         }
7078
7079         if (t->nstids) {
7080                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
7081                     t->stid_base + t->nstids - 1, t->stids_in_use);
7082         }
7083
7084         if (t->nftids) {
7085                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
7086                     t->ftid_base + t->nftids - 1);
7087         }
7088
7089         if (t->netids) {
7090                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
7091                     t->etid_base + t->netids - 1);
7092         }
7093
7094         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
7095             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
7096             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
7097
7098         rc = sbuf_finish(sb);
7099         sbuf_delete(sb);
7100
7101         return (rc);
7102 }
7103
7104 static int
7105 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
7106 {
7107         struct adapter *sc = arg1;
7108         struct sbuf *sb;
7109         int rc;
7110         struct tp_err_stats stats;
7111
7112         rc = sysctl_wire_old_buffer(req, 0);
7113         if (rc != 0)
7114                 return (rc);
7115
7116         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7117         if (sb == NULL)
7118                 return (ENOMEM);
7119
7120         t4_tp_get_err_stats(sc, &stats);
7121
7122         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
7123                       "channel 3\n");
7124         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
7125             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
7126             stats.macInErrs[3]);
7127         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
7128             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
7129             stats.hdrInErrs[3]);
7130         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
7131             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
7132             stats.tcpInErrs[3]);
7133         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
7134             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
7135             stats.tcp6InErrs[3]);
7136         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
7137             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
7138             stats.tnlCongDrops[3]);
7139         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
7140             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
7141             stats.tnlTxDrops[3]);
7142         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
7143             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
7144             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
7145         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
7146             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
7147             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
7148         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
7149             stats.ofldNoNeigh, stats.ofldCongDefer);
7150
7151         rc = sbuf_finish(sb);
7152         sbuf_delete(sb);
7153
7154         return (rc);
7155 }
7156
7157 struct field_desc {
7158         const char *name;
7159         u_int start;
7160         u_int width;
7161 };
7162
7163 static void
7164 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
7165 {
7166         char buf[32];
7167         int line_size = 0;
7168
7169         while (f->name) {
7170                 uint64_t mask = (1ULL << f->width) - 1;
7171                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
7172                     ((uintmax_t)v >> f->start) & mask);
7173
7174                 if (line_size + len >= 79) {
7175                         line_size = 8;
7176                         sbuf_printf(sb, "\n        ");
7177                 }
7178                 sbuf_printf(sb, "%s ", buf);
7179                 line_size += len + 1;
7180                 f++;
7181         }
7182         sbuf_printf(sb, "\n");
7183 }
7184
7185 static struct field_desc tp_la0[] = {
7186         { "RcfOpCodeOut", 60, 4 },
7187         { "State", 56, 4 },
7188         { "WcfState", 52, 4 },
7189         { "RcfOpcSrcOut", 50, 2 },
7190         { "CRxError", 49, 1 },
7191         { "ERxError", 48, 1 },
7192         { "SanityFailed", 47, 1 },
7193         { "SpuriousMsg", 46, 1 },
7194         { "FlushInputMsg", 45, 1 },
7195         { "FlushInputCpl", 44, 1 },
7196         { "RssUpBit", 43, 1 },
7197         { "RssFilterHit", 42, 1 },
7198         { "Tid", 32, 10 },
7199         { "InitTcb", 31, 1 },
7200         { "LineNumber", 24, 7 },
7201         { "Emsg", 23, 1 },
7202         { "EdataOut", 22, 1 },
7203         { "Cmsg", 21, 1 },
7204         { "CdataOut", 20, 1 },
7205         { "EreadPdu", 19, 1 },
7206         { "CreadPdu", 18, 1 },
7207         { "TunnelPkt", 17, 1 },
7208         { "RcfPeerFin", 16, 1 },
7209         { "RcfReasonOut", 12, 4 },
7210         { "TxCchannel", 10, 2 },
7211         { "RcfTxChannel", 8, 2 },
7212         { "RxEchannel", 6, 2 },
7213         { "RcfRxChannel", 5, 1 },
7214         { "RcfDataOutSrdy", 4, 1 },
7215         { "RxDvld", 3, 1 },
7216         { "RxOoDvld", 2, 1 },
7217         { "RxCongestion", 1, 1 },
7218         { "TxCongestion", 0, 1 },
7219         { NULL }
7220 };
7221
7222 static struct field_desc tp_la1[] = {
7223         { "CplCmdIn", 56, 8 },
7224         { "CplCmdOut", 48, 8 },
7225         { "ESynOut", 47, 1 },
7226         { "EAckOut", 46, 1 },
7227         { "EFinOut", 45, 1 },
7228         { "ERstOut", 44, 1 },
7229         { "SynIn", 43, 1 },
7230         { "AckIn", 42, 1 },
7231         { "FinIn", 41, 1 },
7232         { "RstIn", 40, 1 },
7233         { "DataIn", 39, 1 },
7234         { "DataInVld", 38, 1 },
7235         { "PadIn", 37, 1 },
7236         { "RxBufEmpty", 36, 1 },
7237         { "RxDdp", 35, 1 },
7238         { "RxFbCongestion", 34, 1 },
7239         { "TxFbCongestion", 33, 1 },
7240         { "TxPktSumSrdy", 32, 1 },
7241         { "RcfUlpType", 28, 4 },
7242         { "Eread", 27, 1 },
7243         { "Ebypass", 26, 1 },
7244         { "Esave", 25, 1 },
7245         { "Static0", 24, 1 },
7246         { "Cread", 23, 1 },
7247         { "Cbypass", 22, 1 },
7248         { "Csave", 21, 1 },
7249         { "CPktOut", 20, 1 },
7250         { "RxPagePoolFull", 18, 2 },
7251         { "RxLpbkPkt", 17, 1 },
7252         { "TxLpbkPkt", 16, 1 },
7253         { "RxVfValid", 15, 1 },
7254         { "SynLearned", 14, 1 },
7255         { "SetDelEntry", 13, 1 },
7256         { "SetInvEntry", 12, 1 },
7257         { "CpcmdDvld", 11, 1 },
7258         { "CpcmdSave", 10, 1 },
7259         { "RxPstructsFull", 8, 2 },
7260         { "EpcmdDvld", 7, 1 },
7261         { "EpcmdFlush", 6, 1 },
7262         { "EpcmdTrimPrefix", 5, 1 },
7263         { "EpcmdTrimPostfix", 4, 1 },
7264         { "ERssIp4Pkt", 3, 1 },
7265         { "ERssIp6Pkt", 2, 1 },
7266         { "ERssTcpUdpPkt", 1, 1 },
7267         { "ERssFceFipPkt", 0, 1 },
7268         { NULL }
7269 };
7270
7271 static struct field_desc tp_la2[] = {
7272         { "CplCmdIn", 56, 8 },
7273         { "MpsVfVld", 55, 1 },
7274         { "MpsPf", 52, 3 },
7275         { "MpsVf", 44, 8 },
7276         { "SynIn", 43, 1 },
7277         { "AckIn", 42, 1 },
7278         { "FinIn", 41, 1 },
7279         { "RstIn", 40, 1 },
7280         { "DataIn", 39, 1 },
7281         { "DataInVld", 38, 1 },
7282         { "PadIn", 37, 1 },
7283         { "RxBufEmpty", 36, 1 },
7284         { "RxDdp", 35, 1 },
7285         { "RxFbCongestion", 34, 1 },
7286         { "TxFbCongestion", 33, 1 },
7287         { "TxPktSumSrdy", 32, 1 },
7288         { "RcfUlpType", 28, 4 },
7289         { "Eread", 27, 1 },
7290         { "Ebypass", 26, 1 },
7291         { "Esave", 25, 1 },
7292         { "Static0", 24, 1 },
7293         { "Cread", 23, 1 },
7294         { "Cbypass", 22, 1 },
7295         { "Csave", 21, 1 },
7296         { "CPktOut", 20, 1 },
7297         { "RxPagePoolFull", 18, 2 },
7298         { "RxLpbkPkt", 17, 1 },
7299         { "TxLpbkPkt", 16, 1 },
7300         { "RxVfValid", 15, 1 },
7301         { "SynLearned", 14, 1 },
7302         { "SetDelEntry", 13, 1 },
7303         { "SetInvEntry", 12, 1 },
7304         { "CpcmdDvld", 11, 1 },
7305         { "CpcmdSave", 10, 1 },
7306         { "RxPstructsFull", 8, 2 },
7307         { "EpcmdDvld", 7, 1 },
7308         { "EpcmdFlush", 6, 1 },
7309         { "EpcmdTrimPrefix", 5, 1 },
7310         { "EpcmdTrimPostfix", 4, 1 },
7311         { "ERssIp4Pkt", 3, 1 },
7312         { "ERssIp6Pkt", 2, 1 },
7313         { "ERssTcpUdpPkt", 1, 1 },
7314         { "ERssFceFipPkt", 0, 1 },
7315         { NULL }
7316 };
7317
7318 static void
7319 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
7320 {
7321
7322         field_desc_show(sb, *p, tp_la0);
7323 }
7324
7325 static void
7326 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
7327 {
7328
7329         if (idx)
7330                 sbuf_printf(sb, "\n");
7331         field_desc_show(sb, p[0], tp_la0);
7332         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7333                 field_desc_show(sb, p[1], tp_la0);
7334 }
7335
7336 static void
7337 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
7338 {
7339
7340         if (idx)
7341                 sbuf_printf(sb, "\n");
7342         field_desc_show(sb, p[0], tp_la0);
7343         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7344                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7345 }
7346
7347 static int
7348 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7349 {
7350         struct adapter *sc = arg1;
7351         struct sbuf *sb;
7352         uint64_t *buf, *p;
7353         int rc;
7354         u_int i, inc;
7355         void (*show_func)(struct sbuf *, uint64_t *, int);
7356
7357         rc = sysctl_wire_old_buffer(req, 0);
7358         if (rc != 0)
7359                 return (rc);
7360
7361         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7362         if (sb == NULL)
7363                 return (ENOMEM);
7364
7365         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7366
7367         t4_tp_read_la(sc, buf, NULL);
7368         p = buf;
7369
7370         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7371         case 2:
7372                 inc = 2;
7373                 show_func = tp_la_show2;
7374                 break;
7375         case 3:
7376                 inc = 2;
7377                 show_func = tp_la_show3;
7378                 break;
7379         default:
7380                 inc = 1;
7381                 show_func = tp_la_show;
7382         }
7383
7384         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7385                 (*show_func)(sb, p, i);
7386
7387         rc = sbuf_finish(sb);
7388         sbuf_delete(sb);
7389         free(buf, M_CXGBE);
7390         return (rc);
7391 }
7392
7393 static int
7394 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7395 {
7396         struct adapter *sc = arg1;
7397         struct sbuf *sb;
7398         int rc;
7399         u64 nrate[NCHAN], orate[NCHAN];
7400
7401         rc = sysctl_wire_old_buffer(req, 0);
7402         if (rc != 0)
7403                 return (rc);
7404
7405         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7406         if (sb == NULL)
7407                 return (ENOMEM);
7408
7409         t4_get_chan_txrate(sc, nrate, orate);
7410         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
7411                  "channel 3\n");
7412         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7413             nrate[0], nrate[1], nrate[2], nrate[3]);
7414         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7415             orate[0], orate[1], orate[2], orate[3]);
7416
7417         rc = sbuf_finish(sb);
7418         sbuf_delete(sb);
7419
7420         return (rc);
7421 }
7422
7423 static int
7424 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7425 {
7426         struct adapter *sc = arg1;
7427         struct sbuf *sb;
7428         uint32_t *buf, *p;
7429         int rc, i;
7430
7431         rc = sysctl_wire_old_buffer(req, 0);
7432         if (rc != 0)
7433                 return (rc);
7434
7435         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7436         if (sb == NULL)
7437                 return (ENOMEM);
7438
7439         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7440             M_ZERO | M_WAITOK);
7441
7442         t4_ulprx_read_la(sc, buf);
7443         p = buf;
7444
7445         sbuf_printf(sb, "      Pcmd        Type   Message"
7446             "                Data");
7447         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7448                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7449                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7450         }
7451
7452         rc = sbuf_finish(sb);
7453         sbuf_delete(sb);
7454         free(buf, M_CXGBE);
7455         return (rc);
7456 }
7457
7458 static int
7459 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7460 {
7461         struct adapter *sc = arg1;
7462         struct sbuf *sb;
7463         int rc, v;
7464
7465         rc = sysctl_wire_old_buffer(req, 0);
7466         if (rc != 0)
7467                 return (rc);
7468
7469         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7470         if (sb == NULL)
7471                 return (ENOMEM);
7472
7473         v = t4_read_reg(sc, A_SGE_STAT_CFG);
7474         if (G_STATSOURCE_T5(v) == 7) {
7475                 if (G_STATMODE(v) == 0) {
7476                         sbuf_printf(sb, "total %d, incomplete %d",
7477                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7478                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7479                 } else if (G_STATMODE(v) == 1) {
7480                         sbuf_printf(sb, "total %d, data overflow %d",
7481                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7482                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7483                 }
7484         }
7485         rc = sbuf_finish(sb);
7486         sbuf_delete(sb);
7487
7488         return (rc);
7489 }
7490 #endif
7491
7492 static uint32_t
7493 fconf_to_mode(uint32_t fconf)
7494 {
7495         uint32_t mode;
7496
7497         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7498             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7499
7500         if (fconf & F_FRAGMENTATION)
7501                 mode |= T4_FILTER_IP_FRAGMENT;
7502
7503         if (fconf & F_MPSHITTYPE)
7504                 mode |= T4_FILTER_MPS_HIT_TYPE;
7505
7506         if (fconf & F_MACMATCH)
7507                 mode |= T4_FILTER_MAC_IDX;
7508
7509         if (fconf & F_ETHERTYPE)
7510                 mode |= T4_FILTER_ETH_TYPE;
7511
7512         if (fconf & F_PROTOCOL)
7513                 mode |= T4_FILTER_IP_PROTO;
7514
7515         if (fconf & F_TOS)
7516                 mode |= T4_FILTER_IP_TOS;
7517
7518         if (fconf & F_VLAN)
7519                 mode |= T4_FILTER_VLAN;
7520
7521         if (fconf & F_VNIC_ID)
7522                 mode |= T4_FILTER_VNIC;
7523
7524         if (fconf & F_PORT)
7525                 mode |= T4_FILTER_PORT;
7526
7527         if (fconf & F_FCOE)
7528                 mode |= T4_FILTER_FCoE;
7529
7530         return (mode);
7531 }
7532
7533 static uint32_t
7534 mode_to_fconf(uint32_t mode)
7535 {
7536         uint32_t fconf = 0;
7537
7538         if (mode & T4_FILTER_IP_FRAGMENT)
7539                 fconf |= F_FRAGMENTATION;
7540
7541         if (mode & T4_FILTER_MPS_HIT_TYPE)
7542                 fconf |= F_MPSHITTYPE;
7543
7544         if (mode & T4_FILTER_MAC_IDX)
7545                 fconf |= F_MACMATCH;
7546
7547         if (mode & T4_FILTER_ETH_TYPE)
7548                 fconf |= F_ETHERTYPE;
7549
7550         if (mode & T4_FILTER_IP_PROTO)
7551                 fconf |= F_PROTOCOL;
7552
7553         if (mode & T4_FILTER_IP_TOS)
7554                 fconf |= F_TOS;
7555
7556         if (mode & T4_FILTER_VLAN)
7557                 fconf |= F_VLAN;
7558
7559         if (mode & T4_FILTER_VNIC)
7560                 fconf |= F_VNIC_ID;
7561
7562         if (mode & T4_FILTER_PORT)
7563                 fconf |= F_PORT;
7564
7565         if (mode & T4_FILTER_FCoE)
7566                 fconf |= F_FCOE;
7567
7568         return (fconf);
7569 }
7570
7571 static uint32_t
7572 fspec_to_fconf(struct t4_filter_specification *fs)
7573 {
7574         uint32_t fconf = 0;
7575
7576         if (fs->val.frag || fs->mask.frag)
7577                 fconf |= F_FRAGMENTATION;
7578
7579         if (fs->val.matchtype || fs->mask.matchtype)
7580                 fconf |= F_MPSHITTYPE;
7581
7582         if (fs->val.macidx || fs->mask.macidx)
7583                 fconf |= F_MACMATCH;
7584
7585         if (fs->val.ethtype || fs->mask.ethtype)
7586                 fconf |= F_ETHERTYPE;
7587
7588         if (fs->val.proto || fs->mask.proto)
7589                 fconf |= F_PROTOCOL;
7590
7591         if (fs->val.tos || fs->mask.tos)
7592                 fconf |= F_TOS;
7593
7594         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7595                 fconf |= F_VLAN;
7596
7597         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7598                 fconf |= F_VNIC_ID;
7599
7600         if (fs->val.iport || fs->mask.iport)
7601                 fconf |= F_PORT;
7602
7603         if (fs->val.fcoe || fs->mask.fcoe)
7604                 fconf |= F_FCOE;
7605
7606         return (fconf);
7607 }
7608
7609 static int
7610 get_filter_mode(struct adapter *sc, uint32_t *mode)
7611 {
7612         int rc;
7613         uint32_t fconf;
7614
7615         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7616             "t4getfm");
7617         if (rc)
7618                 return (rc);
7619
7620         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7621             A_TP_VLAN_PRI_MAP);
7622
7623         if (sc->params.tp.vlan_pri_map != fconf) {
7624                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7625                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7626                     fconf);
7627         }
7628
7629         *mode = fconf_to_mode(fconf);
7630
7631         end_synchronized_op(sc, LOCK_HELD);
7632         return (0);
7633 }
7634
7635 static int
7636 set_filter_mode(struct adapter *sc, uint32_t mode)
7637 {
7638         uint32_t fconf;
7639         int rc;
7640
7641         fconf = mode_to_fconf(mode);
7642
7643         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7644             "t4setfm");
7645         if (rc)
7646                 return (rc);
7647
7648         if (sc->tids.ftids_in_use > 0) {
7649                 rc = EBUSY;
7650                 goto done;
7651         }
7652
7653 #ifdef TCP_OFFLOAD
7654         if (uld_active(sc, ULD_TOM)) {
7655                 rc = EBUSY;
7656                 goto done;
7657         }
7658 #endif
7659
7660         rc = -t4_set_filter_mode(sc, fconf);
7661 done:
7662         end_synchronized_op(sc, LOCK_HELD);
7663         return (rc);
7664 }
7665
7666 static inline uint64_t
7667 get_filter_hits(struct adapter *sc, uint32_t fid)
7668 {
7669         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7670         uint64_t hits;
7671
7672         memwin_info(sc, 0, &mw_base, NULL);
7673         off = position_memwin(sc, 0,
7674             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7675         if (is_t4(sc)) {
7676                 hits = t4_read_reg64(sc, mw_base + off + 16);
7677                 hits = be64toh(hits);
7678         } else {
7679                 hits = t4_read_reg(sc, mw_base + off + 24);
7680                 hits = be32toh(hits);
7681         }
7682
7683         return (hits);
7684 }
7685
7686 static int
7687 get_filter(struct adapter *sc, struct t4_filter *t)
7688 {
7689         int i, rc, nfilters = sc->tids.nftids;
7690         struct filter_entry *f;
7691
7692         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7693             "t4getf");
7694         if (rc)
7695                 return (rc);
7696
7697         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7698             t->idx >= nfilters) {
7699                 t->idx = 0xffffffff;
7700                 goto done;
7701         }
7702
7703         f = &sc->tids.ftid_tab[t->idx];
7704         for (i = t->idx; i < nfilters; i++, f++) {
7705                 if (f->valid) {
7706                         t->idx = i;
7707                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7708                         t->smtidx = f->smtidx;
7709                         if (f->fs.hitcnts)
7710                                 t->hits = get_filter_hits(sc, t->idx);
7711                         else
7712                                 t->hits = UINT64_MAX;
7713                         t->fs = f->fs;
7714
7715                         goto done;
7716                 }
7717         }
7718
7719         t->idx = 0xffffffff;
7720 done:
7721         end_synchronized_op(sc, LOCK_HELD);
7722         return (0);
7723 }
7724
7725 static int
7726 set_filter(struct adapter *sc, struct t4_filter *t)
7727 {
7728         unsigned int nfilters, nports;
7729         struct filter_entry *f;
7730         int i, rc;
7731
7732         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7733         if (rc)
7734                 return (rc);
7735
7736         nfilters = sc->tids.nftids;
7737         nports = sc->params.nports;
7738
7739         if (nfilters == 0) {
7740                 rc = ENOTSUP;
7741                 goto done;
7742         }
7743
7744         if (!(sc->flags & FULL_INIT_DONE)) {
7745                 rc = EAGAIN;
7746                 goto done;
7747         }
7748
7749         if (t->idx >= nfilters) {
7750                 rc = EINVAL;
7751                 goto done;
7752         }
7753
7754         /* Validate against the global filter mode */
7755         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7756             sc->params.tp.vlan_pri_map) {
7757                 rc = E2BIG;
7758                 goto done;
7759         }
7760
7761         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7762                 rc = EINVAL;
7763                 goto done;
7764         }
7765
7766         if (t->fs.val.iport >= nports) {
7767                 rc = EINVAL;
7768                 goto done;
7769         }
7770
7771         /* Can't specify an iq if not steering to it */
7772         if (!t->fs.dirsteer && t->fs.iq) {
7773                 rc = EINVAL;
7774                 goto done;
7775         }
7776
7777         /* IPv6 filter idx must be 4 aligned */
7778         if (t->fs.type == 1 &&
7779             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7780                 rc = EINVAL;
7781                 goto done;
7782         }
7783
7784         if (sc->tids.ftid_tab == NULL) {
7785                 KASSERT(sc->tids.ftids_in_use == 0,
7786                     ("%s: no memory allocated but filters_in_use > 0",
7787                     __func__));
7788
7789                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7790                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7791                 if (sc->tids.ftid_tab == NULL) {
7792                         rc = ENOMEM;
7793                         goto done;
7794                 }
7795                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7796         }
7797
7798         for (i = 0; i < 4; i++) {
7799                 f = &sc->tids.ftid_tab[t->idx + i];
7800
7801                 if (f->pending || f->valid) {
7802                         rc = EBUSY;
7803                         goto done;
7804                 }
7805                 if (f->locked) {
7806                         rc = EPERM;
7807                         goto done;
7808                 }
7809
7810                 if (t->fs.type == 0)
7811                         break;
7812         }
7813
7814         f = &sc->tids.ftid_tab[t->idx];
7815         f->fs = t->fs;
7816
7817         rc = set_filter_wr(sc, t->idx);
7818 done:
7819         end_synchronized_op(sc, 0);
7820
7821         if (rc == 0) {
7822                 mtx_lock(&sc->tids.ftid_lock);
7823                 for (;;) {
7824                         if (f->pending == 0) {
7825                                 rc = f->valid ? 0 : EIO;
7826                                 break;
7827                         }
7828
7829                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7830                             PCATCH, "t4setfw", 0)) {
7831                                 rc = EINPROGRESS;
7832                                 break;
7833                         }
7834                 }
7835                 mtx_unlock(&sc->tids.ftid_lock);
7836         }
7837         return (rc);
7838 }
7839
7840 static int
7841 del_filter(struct adapter *sc, struct t4_filter *t)
7842 {
7843         unsigned int nfilters;
7844         struct filter_entry *f;
7845         int rc;
7846
7847         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7848         if (rc)
7849                 return (rc);
7850
7851         nfilters = sc->tids.nftids;
7852
7853         if (nfilters == 0) {
7854                 rc = ENOTSUP;
7855                 goto done;
7856         }
7857
7858         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7859             t->idx >= nfilters) {
7860                 rc = EINVAL;
7861                 goto done;
7862         }
7863
7864         if (!(sc->flags & FULL_INIT_DONE)) {
7865                 rc = EAGAIN;
7866                 goto done;
7867         }
7868
7869         f = &sc->tids.ftid_tab[t->idx];
7870
7871         if (f->pending) {
7872                 rc = EBUSY;
7873                 goto done;
7874         }
7875         if (f->locked) {
7876                 rc = EPERM;
7877                 goto done;
7878         }
7879
7880         if (f->valid) {
7881                 t->fs = f->fs;  /* extra info for the caller */
7882                 rc = del_filter_wr(sc, t->idx);
7883         }
7884
7885 done:
7886         end_synchronized_op(sc, 0);
7887
7888         if (rc == 0) {
7889                 mtx_lock(&sc->tids.ftid_lock);
7890                 for (;;) {
7891                         if (f->pending == 0) {
7892                                 rc = f->valid ? EIO : 0;
7893                                 break;
7894                         }
7895
7896                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7897                             PCATCH, "t4delfw", 0)) {
7898                                 rc = EINPROGRESS;
7899                                 break;
7900                         }
7901                 }
7902                 mtx_unlock(&sc->tids.ftid_lock);
7903         }
7904
7905         return (rc);
7906 }
7907
7908 static void
7909 clear_filter(struct filter_entry *f)
7910 {
7911         if (f->l2t)
7912                 t4_l2t_release(f->l2t);
7913
7914         bzero(f, sizeof (*f));
7915 }
7916
7917 static int
7918 set_filter_wr(struct adapter *sc, int fidx)
7919 {
7920         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7921         struct fw_filter_wr *fwr;
7922         unsigned int ftid;
7923         struct wrq_cookie cookie;
7924
7925         ASSERT_SYNCHRONIZED_OP(sc);
7926
7927         if (f->fs.newdmac || f->fs.newvlan) {
7928                 /* This filter needs an L2T entry; allocate one. */
7929                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7930                 if (f->l2t == NULL)
7931                         return (EAGAIN);
7932                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7933                     f->fs.dmac)) {
7934                         t4_l2t_release(f->l2t);
7935                         f->l2t = NULL;
7936                         return (ENOMEM);
7937                 }
7938         }
7939
7940         ftid = sc->tids.ftid_base + fidx;
7941
7942         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7943         if (fwr == NULL)
7944                 return (ENOMEM);
7945         bzero(fwr, sizeof(*fwr));
7946
7947         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7948         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7949         fwr->tid_to_iq =
7950             htobe32(V_FW_FILTER_WR_TID(ftid) |
7951                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7952                 V_FW_FILTER_WR_NOREPLY(0) |
7953                 V_FW_FILTER_WR_IQ(f->fs.iq));
7954         fwr->del_filter_to_l2tix =
7955             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7956                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7957                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7958                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7959                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7960                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7961                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7962                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7963                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7964                     f->fs.newvlan == VLAN_REWRITE) |
7965                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7966                     f->fs.newvlan == VLAN_REWRITE) |
7967                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7968                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7969                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7970                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7971         fwr->ethtype = htobe16(f->fs.val.ethtype);
7972         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7973         fwr->frag_to_ovlan_vldm =
7974             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7975                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7976                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7977                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7978                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7979                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7980         fwr->smac_sel = 0;
7981         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7982             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7983         fwr->maci_to_matchtypem =
7984             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7985                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7986                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7987                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7988                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7989                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7990                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7991                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7992         fwr->ptcl = f->fs.val.proto;
7993         fwr->ptclm = f->fs.mask.proto;
7994         fwr->ttyp = f->fs.val.tos;
7995         fwr->ttypm = f->fs.mask.tos;
7996         fwr->ivlan = htobe16(f->fs.val.vlan);
7997         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7998         fwr->ovlan = htobe16(f->fs.val.vnic);
7999         fwr->ovlanm = htobe16(f->fs.mask.vnic);
8000         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
8001         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
8002         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
8003         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
8004         fwr->lp = htobe16(f->fs.val.dport);
8005         fwr->lpm = htobe16(f->fs.mask.dport);
8006         fwr->fp = htobe16(f->fs.val.sport);
8007         fwr->fpm = htobe16(f->fs.mask.sport);
8008         if (f->fs.newsmac)
8009                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
8010
8011         f->pending = 1;
8012         sc->tids.ftids_in_use++;
8013
8014         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8015         return (0);
8016 }
8017
8018 static int
8019 del_filter_wr(struct adapter *sc, int fidx)
8020 {
8021         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8022         struct fw_filter_wr *fwr;
8023         unsigned int ftid;
8024         struct wrq_cookie cookie;
8025
8026         ftid = sc->tids.ftid_base + fidx;
8027
8028         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8029         if (fwr == NULL)
8030                 return (ENOMEM);
8031         bzero(fwr, sizeof (*fwr));
8032
8033         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
8034
8035         f->pending = 1;
8036         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8037         return (0);
8038 }
8039
8040 int
8041 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8042 {
8043         struct adapter *sc = iq->adapter;
8044         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
8045         unsigned int idx = GET_TID(rpl);
8046         unsigned int rc;
8047         struct filter_entry *f;
8048
8049         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
8050             rss->opcode));
8051
8052         if (is_ftid(sc, idx)) {
8053
8054                 idx -= sc->tids.ftid_base;
8055                 f = &sc->tids.ftid_tab[idx];
8056                 rc = G_COOKIE(rpl->cookie);
8057
8058                 mtx_lock(&sc->tids.ftid_lock);
8059                 if (rc == FW_FILTER_WR_FLT_ADDED) {
8060                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
8061                             __func__, idx));
8062                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
8063                         f->pending = 0;  /* asynchronous setup completed */
8064                         f->valid = 1;
8065                 } else {
8066                         if (rc != FW_FILTER_WR_FLT_DELETED) {
8067                                 /* Add or delete failed, display an error */
8068                                 log(LOG_ERR,
8069                                     "filter %u setup failed with error %u\n",
8070                                     idx, rc);
8071                         }
8072
8073                         clear_filter(f);
8074                         sc->tids.ftids_in_use--;
8075                 }
8076                 wakeup(&sc->tids.ftid_tab);
8077                 mtx_unlock(&sc->tids.ftid_lock);
8078         }
8079
8080         return (0);
8081 }
8082
8083 static int
8084 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
8085 {
8086         int rc;
8087
8088         if (cntxt->cid > M_CTXTQID)
8089                 return (EINVAL);
8090
8091         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
8092             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
8093                 return (EINVAL);
8094
8095         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
8096         if (rc)
8097                 return (rc);
8098
8099         if (sc->flags & FW_OK) {
8100                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
8101                     &cntxt->data[0]);
8102                 if (rc == 0)
8103                         goto done;
8104         }
8105
8106         /*
8107          * Read via firmware failed or wasn't even attempted.  Read directly via
8108          * the backdoor.
8109          */
8110         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
8111 done:
8112         end_synchronized_op(sc, 0);
8113         return (rc);
8114 }
8115
8116 static int
8117 load_fw(struct adapter *sc, struct t4_data *fw)
8118 {
8119         int rc;
8120         uint8_t *fw_data;
8121
8122         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
8123         if (rc)
8124                 return (rc);
8125
8126         if (sc->flags & FULL_INIT_DONE) {
8127                 rc = EBUSY;
8128                 goto done;
8129         }
8130
8131         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
8132         if (fw_data == NULL) {
8133                 rc = ENOMEM;
8134                 goto done;
8135         }
8136
8137         rc = copyin(fw->data, fw_data, fw->len);
8138         if (rc == 0)
8139                 rc = -t4_load_fw(sc, fw_data, fw->len);
8140
8141         free(fw_data, M_CXGBE);
8142 done:
8143         end_synchronized_op(sc, 0);
8144         return (rc);
8145 }
8146
8147 static int
8148 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
8149 {
8150         uint32_t addr, off, remaining, i, n;
8151         uint32_t *buf, *b;
8152         uint32_t mw_base, mw_aperture;
8153         int rc;
8154         uint8_t *dst;
8155
8156         rc = validate_mem_range(sc, mr->addr, mr->len);
8157         if (rc != 0)
8158                 return (rc);
8159
8160         memwin_info(sc, win, &mw_base, &mw_aperture);
8161         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
8162         addr = mr->addr;
8163         remaining = mr->len;
8164         dst = (void *)mr->data;
8165
8166         while (remaining) {
8167                 off = position_memwin(sc, win, addr);
8168
8169                 /* number of bytes that we'll copy in the inner loop */
8170                 n = min(remaining, mw_aperture - off);
8171                 for (i = 0; i < n; i += 4)
8172                         *b++ = t4_read_reg(sc, mw_base + off + i);
8173
8174                 rc = copyout(buf, dst, n);
8175                 if (rc != 0)
8176                         break;
8177
8178                 b = buf;
8179                 dst += n;
8180                 remaining -= n;
8181                 addr += n;
8182         }
8183
8184         free(buf, M_CXGBE);
8185         return (rc);
8186 }
8187
8188 static int
8189 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
8190 {
8191         int rc;
8192
8193         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
8194                 return (EINVAL);
8195
8196         if (i2cd->len > sizeof(i2cd->data))
8197                 return (EFBIG);
8198
8199         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
8200         if (rc)
8201                 return (rc);
8202         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
8203             i2cd->offset, i2cd->len, &i2cd->data[0]);
8204         end_synchronized_op(sc, 0);
8205
8206         return (rc);
8207 }
8208
8209 static int
8210 in_range(int val, int lo, int hi)
8211 {
8212
8213         return (val < 0 || (val <= hi && val >= lo));
8214 }
8215
8216 static int
8217 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
8218 {
8219         int fw_subcmd, fw_type, rc;
8220
8221         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
8222         if (rc)
8223                 return (rc);
8224
8225         if (!(sc->flags & FULL_INIT_DONE)) {
8226                 rc = EAGAIN;
8227                 goto done;
8228         }
8229
8230         /*
8231          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
8232          * sub-command and type are in common locations.)
8233          */
8234         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
8235                 fw_subcmd = FW_SCHED_SC_CONFIG;
8236         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
8237                 fw_subcmd = FW_SCHED_SC_PARAMS;
8238         else {
8239                 rc = EINVAL;
8240                 goto done;
8241         }
8242         if (p->type == SCHED_CLASS_TYPE_PACKET)
8243                 fw_type = FW_SCHED_TYPE_PKTSCHED;
8244         else {
8245                 rc = EINVAL;
8246                 goto done;
8247         }
8248
8249         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
8250                 /* Vet our parameters ..*/
8251                 if (p->u.config.minmax < 0) {
8252                         rc = EINVAL;
8253                         goto done;
8254                 }
8255
8256                 /* And pass the request to the firmware ...*/
8257                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
8258                 goto done;
8259         }
8260
8261         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
8262                 int fw_level;
8263                 int fw_mode;
8264                 int fw_rateunit;
8265                 int fw_ratemode;
8266
8267                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
8268                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
8269                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
8270                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
8271                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
8272                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
8273                 else {
8274                         rc = EINVAL;
8275                         goto done;
8276                 }
8277
8278                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
8279                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
8280                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
8281                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
8282                 else {
8283                         rc = EINVAL;
8284                         goto done;
8285                 }
8286
8287                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
8288                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
8289                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
8290                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
8291                 else {
8292                         rc = EINVAL;
8293                         goto done;
8294                 }
8295
8296                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
8297                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
8298                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
8299                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
8300                 else {
8301                         rc = EINVAL;
8302                         goto done;
8303                 }
8304
8305                 /* Vet our parameters ... */
8306                 if (!in_range(p->u.params.channel, 0, 3) ||
8307                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
8308                     !in_range(p->u.params.minrate, 0, 10000000) ||
8309                     !in_range(p->u.params.maxrate, 0, 10000000) ||
8310                     !in_range(p->u.params.weight, 0, 100)) {
8311                         rc = ERANGE;
8312                         goto done;
8313                 }
8314
8315                 /*
8316                  * Translate any unset parameters into the firmware's
8317                  * nomenclature and/or fail the call if the parameters
8318                  * are required ...
8319                  */
8320                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
8321                     p->u.params.channel < 0 || p->u.params.cl < 0) {
8322                         rc = EINVAL;
8323                         goto done;
8324                 }
8325                 if (p->u.params.minrate < 0)
8326                         p->u.params.minrate = 0;
8327                 if (p->u.params.maxrate < 0) {
8328                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8329                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8330                                 rc = EINVAL;
8331                                 goto done;
8332                         } else
8333                                 p->u.params.maxrate = 0;
8334                 }
8335                 if (p->u.params.weight < 0) {
8336                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
8337                                 rc = EINVAL;
8338                                 goto done;
8339                         } else
8340                                 p->u.params.weight = 0;
8341                 }
8342                 if (p->u.params.pktsize < 0) {
8343                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8344                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8345                                 rc = EINVAL;
8346                                 goto done;
8347                         } else
8348                                 p->u.params.pktsize = 0;
8349                 }
8350
8351                 /* See what the firmware thinks of the request ... */
8352                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
8353                     fw_rateunit, fw_ratemode, p->u.params.channel,
8354                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
8355                     p->u.params.weight, p->u.params.pktsize, 1);
8356                 goto done;
8357         }
8358
8359         rc = EINVAL;
8360 done:
8361         end_synchronized_op(sc, 0);
8362         return (rc);
8363 }
8364
8365 static int
8366 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8367 {
8368         struct port_info *pi = NULL;
8369         struct vi_info *vi;
8370         struct sge_txq *txq;
8371         uint32_t fw_mnem, fw_queue, fw_class;
8372         int i, rc;
8373
8374         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8375         if (rc)
8376                 return (rc);
8377
8378         if (!(sc->flags & FULL_INIT_DONE)) {
8379                 rc = EAGAIN;
8380                 goto done;
8381         }
8382
8383         if (p->port >= sc->params.nports) {
8384                 rc = EINVAL;
8385                 goto done;
8386         }
8387
8388         /* XXX: Only supported for the main VI. */
8389         pi = sc->port[p->port];
8390         vi = &pi->vi[0];
8391         if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
8392                 rc = EINVAL;
8393                 goto done;
8394         }
8395
8396         /*
8397          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8398          * Scheduling Class in this case).
8399          */
8400         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8401             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
8402         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
8403
8404         /*
8405          * If op.queue is non-negative, then we're only changing the scheduling
8406          * on a single specified TX queue.
8407          */
8408         if (p->queue >= 0) {
8409                 txq = &sc->sge.txq[vi->first_txq + p->queue];
8410                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8411                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8412                     &fw_class);
8413                 goto done;
8414         }
8415
8416         /*
8417          * Change the scheduling on all the TX queues for the
8418          * interface.
8419          */
8420         for_each_txq(vi, i, txq) {
8421                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8422                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8423                     &fw_class);
8424                 if (rc)
8425                         goto done;
8426         }
8427
8428         rc = 0;
8429 done:
8430         end_synchronized_op(sc, 0);
8431         return (rc);
8432 }
8433
8434 int
8435 t4_os_find_pci_capability(struct adapter *sc, int cap)
8436 {
8437         int i;
8438
8439         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
8440 }
8441
8442 int
8443 t4_os_pci_save_state(struct adapter *sc)
8444 {
8445         device_t dev;
8446         struct pci_devinfo *dinfo;
8447
8448         dev = sc->dev;
8449         dinfo = device_get_ivars(dev);
8450
8451         pci_cfg_save(dev, dinfo, 0);
8452         return (0);
8453 }
8454
8455 int
8456 t4_os_pci_restore_state(struct adapter *sc)
8457 {
8458         device_t dev;
8459         struct pci_devinfo *dinfo;
8460
8461         dev = sc->dev;
8462         dinfo = device_get_ivars(dev);
8463
8464         pci_cfg_restore(dev, dinfo);
8465         return (0);
8466 }
8467
8468 void
8469 t4_os_portmod_changed(const struct adapter *sc, int idx)
8470 {
8471         struct port_info *pi = sc->port[idx];
8472         struct vi_info *vi;
8473         struct ifnet *ifp;
8474         int v;
8475         static const char *mod_str[] = {
8476                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8477         };
8478
8479         for_each_vi(pi, v, vi) {
8480                 build_medialist(pi, &vi->media);
8481         }
8482
8483         ifp = pi->vi[0].ifp;
8484         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8485                 if_printf(ifp, "transceiver unplugged.\n");
8486         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8487                 if_printf(ifp, "unknown transceiver inserted.\n");
8488         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8489                 if_printf(ifp, "unsupported transceiver inserted.\n");
8490         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8491                 if_printf(ifp, "%s transceiver inserted.\n",
8492                     mod_str[pi->mod_type]);
8493         } else {
8494                 if_printf(ifp, "transceiver (type %d) inserted.\n",
8495                     pi->mod_type);
8496         }
8497 }
8498
8499 void
8500 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8501 {
8502         struct port_info *pi = sc->port[idx];
8503         struct vi_info *vi;
8504         struct ifnet *ifp;
8505         int v;
8506
8507         if (link_stat)
8508                 pi->linkdnrc = -1;
8509         else {
8510                 if (reason >= 0)
8511                         pi->linkdnrc = reason;
8512         }
8513         for_each_vi(pi, v, vi) {
8514                 ifp = vi->ifp;
8515                 if (ifp == NULL)
8516                         continue;
8517
8518                 if (link_stat) {
8519                         ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8520                         if_link_state_change(ifp, LINK_STATE_UP);
8521                 } else {
8522                         if_link_state_change(ifp, LINK_STATE_DOWN);
8523                 }
8524         }
8525 }
8526
8527 void
8528 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8529 {
8530         struct adapter *sc;
8531
8532         sx_slock(&t4_list_lock);
8533         SLIST_FOREACH(sc, &t4_list, link) {
8534                 /*
8535                  * func should not make any assumptions about what state sc is
8536                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8537                  */
8538                 func(sc, arg);
8539         }
8540         sx_sunlock(&t4_list_lock);
8541 }
8542
8543 static int
8544 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8545 {
8546        return (0);
8547 }
8548
8549 static int
8550 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8551 {
8552        return (0);
8553 }
8554
8555 static int
8556 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8557     struct thread *td)
8558 {
8559         int rc;
8560         struct adapter *sc = dev->si_drv1;
8561
8562         rc = priv_check(td, PRIV_DRIVER);
8563         if (rc != 0)
8564                 return (rc);
8565
8566         switch (cmd) {
8567         case CHELSIO_T4_GETREG: {
8568                 struct t4_reg *edata = (struct t4_reg *)data;
8569
8570                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8571                         return (EFAULT);
8572
8573                 if (edata->size == 4)
8574                         edata->val = t4_read_reg(sc, edata->addr);
8575                 else if (edata->size == 8)
8576                         edata->val = t4_read_reg64(sc, edata->addr);
8577                 else
8578                         return (EINVAL);
8579
8580                 break;
8581         }
8582         case CHELSIO_T4_SETREG: {
8583                 struct t4_reg *edata = (struct t4_reg *)data;
8584
8585                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8586                         return (EFAULT);
8587
8588                 if (edata->size == 4) {
8589                         if (edata->val & 0xffffffff00000000)
8590                                 return (EINVAL);
8591                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8592                 } else if (edata->size == 8)
8593                         t4_write_reg64(sc, edata->addr, edata->val);
8594                 else
8595                         return (EINVAL);
8596                 break;
8597         }
8598         case CHELSIO_T4_REGDUMP: {
8599                 struct t4_regdump *regs = (struct t4_regdump *)data;
8600                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8601                 uint8_t *buf;
8602
8603                 if (regs->len < reglen) {
8604                         regs->len = reglen; /* hint to the caller */
8605                         return (ENOBUFS);
8606                 }
8607
8608                 regs->len = reglen;
8609                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8610                 t4_get_regs(sc, regs, buf);
8611                 rc = copyout(buf, regs->data, reglen);
8612                 free(buf, M_CXGBE);
8613                 break;
8614         }
8615         case CHELSIO_T4_GET_FILTER_MODE:
8616                 rc = get_filter_mode(sc, (uint32_t *)data);
8617                 break;
8618         case CHELSIO_T4_SET_FILTER_MODE:
8619                 rc = set_filter_mode(sc, *(uint32_t *)data);
8620                 break;
8621         case CHELSIO_T4_GET_FILTER:
8622                 rc = get_filter(sc, (struct t4_filter *)data);
8623                 break;
8624         case CHELSIO_T4_SET_FILTER:
8625                 rc = set_filter(sc, (struct t4_filter *)data);
8626                 break;
8627         case CHELSIO_T4_DEL_FILTER:
8628                 rc = del_filter(sc, (struct t4_filter *)data);
8629                 break;
8630         case CHELSIO_T4_GET_SGE_CONTEXT:
8631                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8632                 break;
8633         case CHELSIO_T4_LOAD_FW:
8634                 rc = load_fw(sc, (struct t4_data *)data);
8635                 break;
8636         case CHELSIO_T4_GET_MEM:
8637                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8638                 break;
8639         case CHELSIO_T4_GET_I2C:
8640                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8641                 break;
8642         case CHELSIO_T4_CLEAR_STATS: {
8643                 int i, v;
8644                 u_int port_id = *(uint32_t *)data;
8645                 struct port_info *pi;
8646                 struct vi_info *vi;
8647
8648                 if (port_id >= sc->params.nports)
8649                         return (EINVAL);
8650                 pi = sc->port[port_id];
8651
8652                 /* MAC stats */
8653                 t4_clr_port_stats(sc, pi->tx_chan);
8654                 pi->tx_parse_error = 0;
8655                 mtx_lock(&sc->regwin_lock);
8656                 for_each_vi(pi, v, vi) {
8657                         if (vi->flags & VI_INIT_DONE)
8658                                 t4_clr_vi_stats(sc, vi->viid);
8659                 }
8660                 mtx_unlock(&sc->regwin_lock);
8661
8662                 /*
8663                  * Since this command accepts a port, clear stats for
8664                  * all VIs on this port.
8665                  */
8666                 for_each_vi(pi, v, vi) {
8667                         if (vi->flags & VI_INIT_DONE) {
8668                                 struct sge_rxq *rxq;
8669                                 struct sge_txq *txq;
8670                                 struct sge_wrq *wrq;
8671
8672                                 for_each_rxq(vi, i, rxq) {
8673 #if defined(INET) || defined(INET6)
8674                                         rxq->lro.lro_queued = 0;
8675                                         rxq->lro.lro_flushed = 0;
8676 #endif
8677                                         rxq->rxcsum = 0;
8678                                         rxq->vlan_extraction = 0;
8679                                 }
8680
8681                                 for_each_txq(vi, i, txq) {
8682                                         txq->txcsum = 0;
8683                                         txq->tso_wrs = 0;
8684                                         txq->vlan_insertion = 0;
8685                                         txq->imm_wrs = 0;
8686                                         txq->sgl_wrs = 0;
8687                                         txq->txpkt_wrs = 0;
8688                                         txq->txpkts0_wrs = 0;
8689                                         txq->txpkts1_wrs = 0;
8690                                         txq->txpkts0_pkts = 0;
8691                                         txq->txpkts1_pkts = 0;
8692                                         mp_ring_reset_stats(txq->r);
8693                                 }
8694
8695 #ifdef TCP_OFFLOAD
8696                                 /* nothing to clear for each ofld_rxq */
8697
8698                                 for_each_ofld_txq(vi, i, wrq) {
8699                                         wrq->tx_wrs_direct = 0;
8700                                         wrq->tx_wrs_copied = 0;
8701                                 }
8702 #endif
8703
8704                                 if (IS_MAIN_VI(vi)) {
8705                                         wrq = &sc->sge.ctrlq[pi->port_id];
8706                                         wrq->tx_wrs_direct = 0;
8707                                         wrq->tx_wrs_copied = 0;
8708                                 }
8709                         }
8710                 }
8711                 break;
8712         }
8713         case CHELSIO_T4_SCHED_CLASS:
8714                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8715                 break;
8716         case CHELSIO_T4_SCHED_QUEUE:
8717                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8718                 break;
8719         case CHELSIO_T4_GET_TRACER:
8720                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8721                 break;
8722         case CHELSIO_T4_SET_TRACER:
8723                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8724                 break;
8725         default:
8726                 rc = EINVAL;
8727         }
8728
8729         return (rc);
8730 }
8731
8732 #ifdef TCP_OFFLOAD
8733 void
8734 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8735     const unsigned int *pgsz_order)
8736 {
8737         struct vi_info *vi = ifp->if_softc;
8738         struct adapter *sc = vi->pi->adapter;
8739
8740         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8741         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8742                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8743                 V_HPZ3(pgsz_order[3]));
8744 }
8745
8746 static int
8747 toe_capability(struct vi_info *vi, int enable)
8748 {
8749         int rc;
8750         struct port_info *pi = vi->pi;
8751         struct adapter *sc = pi->adapter;
8752
8753         ASSERT_SYNCHRONIZED_OP(sc);
8754
8755         if (!is_offload(sc))
8756                 return (ENODEV);
8757
8758         if (enable) {
8759                 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
8760                         /* TOE is already enabled. */
8761                         return (0);
8762                 }
8763
8764                 /*
8765                  * We need the port's queues around so that we're able to send
8766                  * and receive CPLs to/from the TOE even if the ifnet for this
8767                  * port has never been UP'd administratively.
8768                  */
8769                 if (!(vi->flags & VI_INIT_DONE)) {
8770                         rc = vi_full_init(vi);
8771                         if (rc)
8772                                 return (rc);
8773                 }
8774                 if (!(pi->vi[0].flags & VI_INIT_DONE)) {
8775                         rc = vi_full_init(&pi->vi[0]);
8776                         if (rc)
8777                                 return (rc);
8778                 }
8779
8780                 if (isset(&sc->offload_map, pi->port_id)) {
8781                         /* TOE is enabled on another VI of this port. */
8782                         pi->uld_vis++;
8783                         return (0);
8784                 }
8785
8786                 if (!uld_active(sc, ULD_TOM)) {
8787                         rc = t4_activate_uld(sc, ULD_TOM);
8788                         if (rc == EAGAIN) {
8789                                 log(LOG_WARNING,
8790                                     "You must kldload t4_tom.ko before trying "
8791                                     "to enable TOE on a cxgbe interface.\n");
8792                         }
8793                         if (rc != 0)
8794                                 return (rc);
8795                         KASSERT(sc->tom_softc != NULL,
8796                             ("%s: TOM activated but softc NULL", __func__));
8797                         KASSERT(uld_active(sc, ULD_TOM),
8798                             ("%s: TOM activated but flag not set", __func__));
8799                 }
8800
8801                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8802                 if (!uld_active(sc, ULD_IWARP))
8803                         (void) t4_activate_uld(sc, ULD_IWARP);
8804                 if (!uld_active(sc, ULD_ISCSI))
8805                         (void) t4_activate_uld(sc, ULD_ISCSI);
8806
8807                 pi->uld_vis++;
8808                 setbit(&sc->offload_map, pi->port_id);
8809         } else {
8810                 pi->uld_vis--;
8811
8812                 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
8813                         return (0);
8814
8815                 KASSERT(uld_active(sc, ULD_TOM),
8816                     ("%s: TOM never initialized?", __func__));
8817                 clrbit(&sc->offload_map, pi->port_id);
8818         }
8819
8820         return (0);
8821 }
8822
8823 /*
8824  * Add an upper layer driver to the global list.
8825  */
8826 int
8827 t4_register_uld(struct uld_info *ui)
8828 {
8829         int rc = 0;
8830         struct uld_info *u;
8831
8832         sx_xlock(&t4_uld_list_lock);
8833         SLIST_FOREACH(u, &t4_uld_list, link) {
8834             if (u->uld_id == ui->uld_id) {
8835                     rc = EEXIST;
8836                     goto done;
8837             }
8838         }
8839
8840         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8841         ui->refcount = 0;
8842 done:
8843         sx_xunlock(&t4_uld_list_lock);
8844         return (rc);
8845 }
8846
8847 int
8848 t4_unregister_uld(struct uld_info *ui)
8849 {
8850         int rc = EINVAL;
8851         struct uld_info *u;
8852
8853         sx_xlock(&t4_uld_list_lock);
8854
8855         SLIST_FOREACH(u, &t4_uld_list, link) {
8856             if (u == ui) {
8857                     if (ui->refcount > 0) {
8858                             rc = EBUSY;
8859                             goto done;
8860                     }
8861
8862                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8863                     rc = 0;
8864                     goto done;
8865             }
8866         }
8867 done:
8868         sx_xunlock(&t4_uld_list_lock);
8869         return (rc);
8870 }
8871
8872 int
8873 t4_activate_uld(struct adapter *sc, int id)
8874 {
8875         int rc;
8876         struct uld_info *ui;
8877
8878         ASSERT_SYNCHRONIZED_OP(sc);
8879
8880         if (id < 0 || id > ULD_MAX)
8881                 return (EINVAL);
8882         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8883
8884         sx_slock(&t4_uld_list_lock);
8885
8886         SLIST_FOREACH(ui, &t4_uld_list, link) {
8887                 if (ui->uld_id == id) {
8888                         if (!(sc->flags & FULL_INIT_DONE)) {
8889                                 rc = adapter_full_init(sc);
8890                                 if (rc != 0)
8891                                         break;
8892                         }
8893
8894                         rc = ui->activate(sc);
8895                         if (rc == 0) {
8896                                 setbit(&sc->active_ulds, id);
8897                                 ui->refcount++;
8898                         }
8899                         break;
8900                 }
8901         }
8902
8903         sx_sunlock(&t4_uld_list_lock);
8904
8905         return (rc);
8906 }
8907
8908 int
8909 t4_deactivate_uld(struct adapter *sc, int id)
8910 {
8911         int rc;
8912         struct uld_info *ui;
8913
8914         ASSERT_SYNCHRONIZED_OP(sc);
8915
8916         if (id < 0 || id > ULD_MAX)
8917                 return (EINVAL);
8918         rc = ENXIO;
8919
8920         sx_slock(&t4_uld_list_lock);
8921
8922         SLIST_FOREACH(ui, &t4_uld_list, link) {
8923                 if (ui->uld_id == id) {
8924                         rc = ui->deactivate(sc);
8925                         if (rc == 0) {
8926                                 clrbit(&sc->active_ulds, id);
8927                                 ui->refcount--;
8928                         }
8929                         break;
8930                 }
8931         }
8932
8933         sx_sunlock(&t4_uld_list_lock);
8934
8935         return (rc);
8936 }
8937
8938 int
8939 uld_active(struct adapter *sc, int uld_id)
8940 {
8941
8942         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8943
8944         return (isset(&sc->active_ulds, uld_id));
8945 }
8946 #endif
8947
8948 /*
8949  * Come up with reasonable defaults for some of the tunables, provided they're
8950  * not set by the user (in which case we'll use the values as is).
8951  */
8952 static void
8953 tweak_tunables(void)
8954 {
8955         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8956
8957         if (t4_ntxq10g < 1) {
8958 #ifdef RSS
8959                 t4_ntxq10g = rss_getnumbuckets();
8960 #else
8961                 t4_ntxq10g = min(nc, NTXQ_10G);
8962 #endif
8963         }
8964
8965         if (t4_ntxq1g < 1) {
8966 #ifdef RSS
8967                 /* XXX: way too many for 1GbE? */
8968                 t4_ntxq1g = rss_getnumbuckets();
8969 #else
8970                 t4_ntxq1g = min(nc, NTXQ_1G);
8971 #endif
8972         }
8973
8974         if (t4_ntxq_vi < 1)
8975                 t4_ntxq_vi = min(nc, NTXQ_VI);
8976
8977         if (t4_nrxq10g < 1) {
8978 #ifdef RSS
8979                 t4_nrxq10g = rss_getnumbuckets();
8980 #else
8981                 t4_nrxq10g = min(nc, NRXQ_10G);
8982 #endif
8983         }
8984
8985         if (t4_nrxq1g < 1) {
8986 #ifdef RSS
8987                 /* XXX: way too many for 1GbE? */
8988                 t4_nrxq1g = rss_getnumbuckets();
8989 #else
8990                 t4_nrxq1g = min(nc, NRXQ_1G);
8991 #endif
8992         }
8993
8994         if (t4_nrxq_vi < 1)
8995                 t4_nrxq_vi = min(nc, NRXQ_VI);
8996
8997 #ifdef TCP_OFFLOAD
8998         if (t4_nofldtxq10g < 1)
8999                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
9000
9001         if (t4_nofldtxq1g < 1)
9002                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
9003
9004         if (t4_nofldtxq_vi < 1)
9005                 t4_nofldtxq_vi = min(nc, NOFLDTXQ_VI);
9006
9007         if (t4_nofldrxq10g < 1)
9008                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
9009
9010         if (t4_nofldrxq1g < 1)
9011                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
9012
9013         if (t4_nofldrxq_vi < 1)
9014                 t4_nofldrxq_vi = min(nc, NOFLDRXQ_VI);
9015
9016         if (t4_toecaps_allowed == -1)
9017                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
9018 #else
9019         if (t4_toecaps_allowed == -1)
9020                 t4_toecaps_allowed = 0;
9021 #endif
9022
9023 #ifdef DEV_NETMAP
9024         if (t4_nnmtxq_vi < 1)
9025                 t4_nnmtxq_vi = min(nc, NNMTXQ_VI);
9026
9027         if (t4_nnmrxq_vi < 1)
9028                 t4_nnmrxq_vi = min(nc, NNMRXQ_VI);
9029 #endif
9030
9031         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
9032                 t4_tmr_idx_10g = TMR_IDX_10G;
9033
9034         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
9035                 t4_pktc_idx_10g = PKTC_IDX_10G;
9036
9037         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
9038                 t4_tmr_idx_1g = TMR_IDX_1G;
9039
9040         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
9041                 t4_pktc_idx_1g = PKTC_IDX_1G;
9042
9043         if (t4_qsize_txq < 128)
9044                 t4_qsize_txq = 128;
9045
9046         if (t4_qsize_rxq < 128)
9047                 t4_qsize_rxq = 128;
9048         while (t4_qsize_rxq & 7)
9049                 t4_qsize_rxq++;
9050
9051         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
9052 }
9053
9054 static struct sx mlu;   /* mod load unload */
9055 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
9056
9057 static int
9058 mod_event(module_t mod, int cmd, void *arg)
9059 {
9060         int rc = 0;
9061         static int loaded = 0;
9062
9063         switch (cmd) {
9064         case MOD_LOAD:
9065                 sx_xlock(&mlu);
9066                 if (loaded++ == 0) {
9067                         t4_sge_modload();
9068                         sx_init(&t4_list_lock, "T4/T5 adapters");
9069                         SLIST_INIT(&t4_list);
9070 #ifdef TCP_OFFLOAD
9071                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
9072                         SLIST_INIT(&t4_uld_list);
9073 #endif
9074                         t4_tracer_modload();
9075                         tweak_tunables();
9076                 }
9077                 sx_xunlock(&mlu);
9078                 break;
9079
9080         case MOD_UNLOAD:
9081                 sx_xlock(&mlu);
9082                 if (--loaded == 0) {
9083                         int tries;
9084
9085                         sx_slock(&t4_list_lock);
9086                         if (!SLIST_EMPTY(&t4_list)) {
9087                                 rc = EBUSY;
9088                                 sx_sunlock(&t4_list_lock);
9089                                 goto done_unload;
9090                         }
9091 #ifdef TCP_OFFLOAD
9092                         sx_slock(&t4_uld_list_lock);
9093                         if (!SLIST_EMPTY(&t4_uld_list)) {
9094                                 rc = EBUSY;
9095                                 sx_sunlock(&t4_uld_list_lock);
9096                                 sx_sunlock(&t4_list_lock);
9097                                 goto done_unload;
9098                         }
9099 #endif
9100                         tries = 0;
9101                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
9102                                 uprintf("%ju clusters with custom free routine "
9103                                     "still is use.\n", t4_sge_extfree_refs());
9104                                 pause("t4unload", 2 * hz);
9105                         }
9106 #ifdef TCP_OFFLOAD
9107                         sx_sunlock(&t4_uld_list_lock);
9108 #endif
9109                         sx_sunlock(&t4_list_lock);
9110
9111                         if (t4_sge_extfree_refs() == 0) {
9112                                 t4_tracer_modunload();
9113 #ifdef TCP_OFFLOAD
9114                                 sx_destroy(&t4_uld_list_lock);
9115 #endif
9116                                 sx_destroy(&t4_list_lock);
9117                                 t4_sge_modunload();
9118                                 loaded = 0;
9119                         } else {
9120                                 rc = EBUSY;
9121                                 loaded++;       /* undo earlier decrement */
9122                         }
9123                 }
9124 done_unload:
9125                 sx_xunlock(&mlu);
9126                 break;
9127         }
9128
9129         return (rc);
9130 }
9131
9132 static devclass_t t4_devclass, t5_devclass;
9133 static devclass_t cxgbe_devclass, cxl_devclass;
9134 static devclass_t vcxgbe_devclass, vcxl_devclass;
9135
9136 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
9137 MODULE_VERSION(t4nex, 1);
9138 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
9139
9140 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
9141 MODULE_VERSION(t5nex, 1);
9142 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
9143
9144 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
9145 MODULE_VERSION(cxgbe, 1);
9146
9147 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
9148 MODULE_VERSION(cxl, 1);
9149
9150 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
9151 MODULE_VERSION(vcxgbe, 1);
9152
9153 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
9154 MODULE_VERSION(vcxl, 1);