sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static int
 574 t4_attach(device_t dev)
 575 {
 576         struct adapter *sc;
 577         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 578         struct intrs_and_queues iaq;
 579         struct sge *s;
 580 #ifdef TCP_OFFLOAD
 581         int ofld_rqidx, ofld_tqidx;
 582 #endif
 583 #ifdef DEV_NETMAP
 584         int nm_rqidx, nm_tqidx;
 585 #endif
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671 #if defined(__i386__)
 672         if ((cpu_feature & CPUID_CX8) == 0) {
 673                 device_printf(dev, "64 bit atomics not available.\n");
 674                 rc = ENOTSUP;
 675                 goto done;
 676         }
 677 #endif
 678
 679         /* Prepare the firmware for operation */
 680         rc = prep_firmware(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = get_params__post_init(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = set_params__post_init(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         rc = map_bar_2(sc);
 693         if (rc != 0)
 694                 goto done; /* error message displayed already */
 695
 696         rc = t4_create_dma_tag(sc);
 697         if (rc != 0)
 698                 goto done; /* error message displayed already */
 699
 700         /*
 701          * First pass over all the ports - allocate VIs and initialize some
 702          * basic parameters like mac address, port type, etc.  We also figure
 703          * out whether a port is 10G or 1G and use that information when
 704          * calculating how many interrupts to attempt to allocate.
 705          */
 706         n10g = n1g = 0;
 707         for_each_port(sc, i) {
 708                 struct port_info *pi;
 709
 710                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 711                 sc->port[i] = pi;
 712
 713                 /* These must be set before t4_port_init */
 714                 pi->adapter = sc;
 715                 pi->port_id = i;
 716
 717                 /* Allocate the vi and initialize parameters like mac addr */
 718                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 719                 if (rc != 0) {
 720                         device_printf(dev, "unable to initialize port %d: %d\n",
 721                             i, rc);
 722                         free(pi, M_CXGBE);
 723                         sc->port[i] = NULL;
 724                         goto done;
 725                 }
 726
 727                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 728                 pi->link_cfg.requested_fc |= t4_pause_settings;
 729                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 730                 pi->link_cfg.fc |= t4_pause_settings;
 731
 732                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 733                 if (rc != 0) {
 734                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 735                         free(pi, M_CXGBE);
 736                         sc->port[i] = NULL;
 737                         goto done;
 738                 }
 739
 740                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 741                     device_get_nameunit(dev), i);
 742                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 743                 sc->chan_map[pi->tx_chan] = i;
 744
 745                 if (is_10G_port(pi) || is_40G_port(pi)) {
 746                         n10g++;
 747                         pi->tmr_idx = t4_tmr_idx_10g;
 748                         pi->pktc_idx = t4_pktc_idx_10g;
 749                 } else {
 750                         n1g++;
 751                         pi->tmr_idx = t4_tmr_idx_1g;
 752                         pi->pktc_idx = t4_pktc_idx_1g;
 753                 }
 754
 755                 pi->xact_addr_filt = -1;
 756                 pi->linkdnrc = -1;
 757
 758                 pi->qsize_rxq = t4_qsize_rxq;
 759                 pi->qsize_txq = t4_qsize_txq;
 760
 761                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 762                 if (pi->dev == NULL) {
 763                         device_printf(dev,
 764                             "failed to add device for port %d.\n", i);
 765                         rc = ENXIO;
 766                         goto done;
 767                 }
 768                 device_set_softc(pi->dev, pi);
 769         }
 770
 771         /*
 772          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 773          */
 774         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 775         if (rc != 0)
 776                 goto done; /* error message displayed already */
 777
 778         sc->intr_type = iaq.intr_type;
 779         sc->intr_count = iaq.nirq;
 780
 781         s = &sc->sge;
 782         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 783         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 784         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 785         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 786         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 787 #ifdef TCP_OFFLOAD
 788         if (is_offload(sc)) {
 789                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 790                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 791                 s->neq += s->nofldtxq + s->nofldrxq;
 792                 s->niq += s->nofldrxq;
 793
 794                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 795                     M_CXGBE, M_ZERO | M_WAITOK);
 796                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 797                     M_CXGBE, M_ZERO | M_WAITOK);
 798         }
 799 #endif
 800 #ifdef DEV_NETMAP
 801         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 802         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 803         s->neq += s->nnmtxq + s->nnmrxq;
 804         s->niq += s->nnmrxq;
 805
 806         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 807             M_CXGBE, M_ZERO | M_WAITOK);
 808         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 809             M_CXGBE, M_ZERO | M_WAITOK);
 810 #endif
 811
 812         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 817             M_ZERO | M_WAITOK);
 818         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 819             M_ZERO | M_WAITOK);
 820         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 821             M_ZERO | M_WAITOK);
 822
 823         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 824             M_ZERO | M_WAITOK);
 825
 826         t4_init_l2t(sc, M_WAITOK);
 827
 828         /*
 829          * Second pass over the ports.  This time we know the number of rx and
 830          * tx queues that each port should get.
 831          */
 832         rqidx = tqidx = 0;
 833 #ifdef TCP_OFFLOAD
 834         ofld_rqidx = ofld_tqidx = 0;
 835 #endif
 836 #ifdef DEV_NETMAP
 837         nm_rqidx = nm_tqidx = 0;
 838 #endif
 839         for_each_port(sc, i) {
 840                 struct port_info *pi = sc->port[i];
 841
 842                 if (pi == NULL)
 843                         continue;
 844
 845                 pi->first_rxq = rqidx;
 846                 pi->first_txq = tqidx;
 847                 if (is_10G_port(pi) || is_40G_port(pi)) {
 848                         pi->flags |= iaq.intr_flags_10g;
 849                         pi->nrxq = iaq.nrxq10g;
 850                         pi->ntxq = iaq.ntxq10g;
 851                 } else {
 852                         pi->flags |= iaq.intr_flags_1g;
 853                         pi->nrxq = iaq.nrxq1g;
 854                         pi->ntxq = iaq.ntxq1g;
 855                 }
 856
 857                 if (pi->ntxq > 1)
 858                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 859                 else
 860                         pi->rsrv_noflowq = 0;
 861
 862                 rqidx += pi->nrxq;
 863                 tqidx += pi->ntxq;
 864 #ifdef TCP_OFFLOAD
 865                 if (is_offload(sc)) {
 866                         pi->first_ofld_rxq = ofld_rqidx;
 867                         pi->first_ofld_txq = ofld_tqidx;
 868                         if (is_10G_port(pi) || is_40G_port(pi)) {
 869                                 pi->nofldrxq = iaq.nofldrxq10g;
 870                                 pi->nofldtxq = iaq.nofldtxq10g;
 871                         } else {
 872                                 pi->nofldrxq = iaq.nofldrxq1g;
 873                                 pi->nofldtxq = iaq.nofldtxq1g;
 874                         }
 875                         ofld_rqidx += pi->nofldrxq;
 876                         ofld_tqidx += pi->nofldtxq;
 877                 }
 878 #endif
 879 #ifdef DEV_NETMAP
 880                 pi->first_nm_rxq = nm_rqidx;
 881                 pi->first_nm_txq = nm_tqidx;
 882                 if (is_10G_port(pi) || is_40G_port(pi)) {
 883                         pi->nnmrxq = iaq.nnmrxq10g;
 884                         pi->nnmtxq = iaq.nnmtxq10g;
 885                 } else {
 886                         pi->nnmrxq = iaq.nnmrxq1g;
 887                         pi->nnmtxq = iaq.nnmtxq1g;
 888                 }
 889                 nm_rqidx += pi->nnmrxq;
 890                 nm_tqidx += pi->nnmtxq;
 891 #endif
 892         }
 893
 894         rc = setup_intr_handlers(sc);
 895         if (rc != 0) {
 896                 device_printf(dev,
 897                     "failed to setup interrupt handlers: %d\n", rc);
 898                 goto done;
 899         }
 900
 901         rc = bus_generic_attach(dev);
 902         if (rc != 0) {
 903                 device_printf(dev,
 904                     "failed to attach all child ports: %d\n", rc);
 905                 goto done;
 906         }
 907
 908         device_printf(dev,
 909             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 910             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 911             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 912             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 913             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 914
 915         t4_set_desc(sc);
 916
 917 done:
 918         if (rc != 0 && sc->cdev) {
 919                 /* cdev was created and so cxgbetool works; recover that way. */
 920                 device_printf(dev,
 921                     "error during attach, adapter is now in recovery mode.\n");
 922                 rc = 0;
 923         }
 924
 925         if (rc != 0)
 926                 t4_detach(dev);
 927         else
 928                 t4_sysctls(sc);
 929
 930         return (rc);
 931 }
 932
 933 /*
 934  * Idempotent
 935  */
 936 static int
 937 t4_detach(device_t dev)
 938 {
 939         struct adapter *sc;
 940         struct port_info *pi;
 941         int i, rc;
 942
 943         sc = device_get_softc(dev);
 944
 945         if (sc->flags & FULL_INIT_DONE)
 946                 t4_intr_disable(sc);
 947
 948         if (sc->cdev) {
 949                 destroy_dev(sc->cdev);
 950                 sc->cdev = NULL;
 951         }
 952
 953         rc = bus_generic_detach(dev);
 954         if (rc) {
 955                 device_printf(dev,
 956                     "failed to detach child devices: %d\n", rc);
 957                 return (rc);
 958         }
 959
 960         for (i = 0; i < sc->intr_count; i++)
 961                 t4_free_irq(sc, &sc->irq[i]);
 962
 963         for (i = 0; i < MAX_NPORTS; i++) {
 964                 pi = sc->port[i];
 965                 if (pi) {
 966                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 967                         if (pi->dev)
 968                                 device_delete_child(dev, pi->dev);
 969
 970                         mtx_destroy(&pi->pi_lock);
 971                         free(pi, M_CXGBE);
 972                 }
 973         }
 974
 975         if (sc->flags & FULL_INIT_DONE)
 976                 adapter_full_uninit(sc);
 977
 978         if (sc->flags & FW_OK)
 979                 t4_fw_bye(sc, sc->mbox);
 980
 981         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 982                 pci_release_msi(dev);
 983
 984         if (sc->regs_res)
 985                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 986                     sc->regs_res);
 987
 988         if (sc->udbs_res)
 989                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 990                     sc->udbs_res);
 991
 992         if (sc->msix_res)
 993                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 994                     sc->msix_res);
 995
 996         if (sc->l2t)
 997                 t4_free_l2t(sc->l2t);
 998
 999 #ifdef TCP_OFFLOAD
1000         free(sc->sge.ofld_rxq, M_CXGBE);
1001         free(sc->sge.ofld_txq, M_CXGBE);
1002 #endif
1003 #ifdef DEV_NETMAP
1004         free(sc->sge.nm_rxq, M_CXGBE);
1005         free(sc->sge.nm_txq, M_CXGBE);
1006 #endif
1007         free(sc->irq, M_CXGBE);
1008         free(sc->sge.rxq, M_CXGBE);
1009         free(sc->sge.txq, M_CXGBE);
1010         free(sc->sge.ctrlq, M_CXGBE);
1011         free(sc->sge.iqmap, M_CXGBE);
1012         free(sc->sge.eqmap, M_CXGBE);
1013         free(sc->tids.ftid_tab, M_CXGBE);
1014         t4_destroy_dma_tag(sc);
1015         if (mtx_initialized(&sc->sc_lock)) {
1016                 sx_xlock(&t4_list_lock);
1017                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1018                 sx_xunlock(&t4_list_lock);
1019                 mtx_destroy(&sc->sc_lock);
1020         }
1021
1022         if (mtx_initialized(&sc->tids.ftid_lock))
1023                 mtx_destroy(&sc->tids.ftid_lock);
1024         if (mtx_initialized(&sc->sfl_lock))
1025                 mtx_destroy(&sc->sfl_lock);
1026         if (mtx_initialized(&sc->ifp_lock))
1027                 mtx_destroy(&sc->ifp_lock);
1028         if (mtx_initialized(&sc->regwin_lock))
1029                 mtx_destroy(&sc->regwin_lock);
1030
1031         bzero(sc, sizeof(*sc));
1032
1033         return (0);
1034 }
1035
1036 static int
1037 cxgbe_probe(device_t dev)
1038 {
1039         char buf[128];
1040         struct port_info *pi = device_get_softc(dev);
1041
1042         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1043         device_set_desc_copy(dev, buf);
1044
1045         return (BUS_PROBE_DEFAULT);
1046 }
1047
1048 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1049     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1050     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1051 #define T4_CAP_ENABLE (T4_CAP)
1052
1053 static int
1054 cxgbe_attach(device_t dev)
1055 {
1056         struct port_info *pi = device_get_softc(dev);
1057         struct ifnet *ifp;
1058         char *s;
1059         int n, o;
1060
1061         /* Allocate an ifnet and set it up */
1062         ifp = if_alloc(IFT_ETHER);
1063         if (ifp == NULL) {
1064                 device_printf(dev, "Cannot allocate ifnet\n");
1065                 return (ENOMEM);
1066         }
1067         pi->ifp = ifp;
1068         ifp->if_softc = pi;
1069
1070         callout_init(&pi->tick, CALLOUT_MPSAFE);
1071
1072         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1073         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1074
1075         ifp->if_init = cxgbe_init;
1076         ifp->if_ioctl = cxgbe_ioctl;
1077         ifp->if_transmit = cxgbe_transmit;
1078         ifp->if_qflush = cxgbe_qflush;
1079
1080         ifp->if_capabilities = T4_CAP;
1081 #ifdef TCP_OFFLOAD
1082         if (is_offload(pi->adapter))
1083                 ifp->if_capabilities |= IFCAP_TOE;
1084 #endif
1085         ifp->if_capenable = T4_CAP_ENABLE;
1086         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1087             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1088
1089         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1090         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1091         ifp->if_hw_tsomaxsegsize = 65536;
1092
1093         /* Initialize ifmedia for this port */
1094         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1095             cxgbe_media_status);
1096         build_medialist(pi, &pi->media);
1097
1098         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1099             EVENTHANDLER_PRI_ANY);
1100
1101         ether_ifattach(ifp, pi->hw_addr);
1102
1103         n = 128;
1104         s = malloc(n, M_CXGBE, M_WAITOK);
1105         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1106         MPASS(n > o);
1107 #ifdef TCP_OFFLOAD
1108         if (is_offload(pi->adapter)) {
1109                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1110                     pi->nofldtxq, pi->nofldrxq);
1111                 MPASS(n > o);
1112         }
1113 #endif
1114 #ifdef DEV_NETMAP
1115         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1116             pi->nnmrxq);
1117         MPASS(n > o);
1118 #endif
1119         device_printf(dev, "%s\n", s);
1120         free(s, M_CXGBE);
1121
1122 #ifdef DEV_NETMAP
1123         /* nm_media handled here to keep implementation private to this file */
1124         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1125             cxgbe_media_status);
1126         build_medialist(pi, &pi->nm_media);
1127         create_netmap_ifnet(pi);        /* logs errors it something fails */
1128 #endif
1129         cxgbe_sysctls(pi);
1130
1131         return (0);
1132 }
1133
1134 static int
1135 cxgbe_detach(device_t dev)
1136 {
1137         struct port_info *pi = device_get_softc(dev);
1138         struct adapter *sc = pi->adapter;
1139         struct ifnet *ifp = pi->ifp;
1140
1141         /* Tell if_ioctl and if_init that the port is going away */
1142         ADAPTER_LOCK(sc);
1143         SET_DOOMED(pi);
1144         wakeup(&sc->flags);
1145         while (IS_BUSY(sc))
1146                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1147         SET_BUSY(sc);
1148 #ifdef INVARIANTS
1149         sc->last_op = "t4detach";
1150         sc->last_op_thr = curthread;
1151 #endif
1152         ADAPTER_UNLOCK(sc);
1153
1154         if (pi->flags & HAS_TRACEQ) {
1155                 sc->traceq = -1;        /* cloner should not create ifnet */
1156                 t4_tracer_port_detach(sc);
1157         }
1158
1159         if (pi->vlan_c)
1160                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1161
1162         PORT_LOCK(pi);
1163         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1164         callout_stop(&pi->tick);
1165         PORT_UNLOCK(pi);
1166         callout_drain(&pi->tick);
1167
1168         /* Let detach proceed even if these fail. */
1169         cxgbe_uninit_synchronized(pi);
1170         port_full_uninit(pi);
1171
1172         ifmedia_removeall(&pi->media);
1173         ether_ifdetach(pi->ifp);
1174         if_free(pi->ifp);
1175
1176 #ifdef DEV_NETMAP
1177         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1178         destroy_netmap_ifnet(pi);
1179 #endif
1180
1181         ADAPTER_LOCK(sc);
1182         CLR_BUSY(sc);
1183         wakeup(&sc->flags);
1184         ADAPTER_UNLOCK(sc);
1185
1186         return (0);
1187 }
1188
1189 static void
1190 cxgbe_init(void *arg)
1191 {
1192         struct port_info *pi = arg;
1193         struct adapter *sc = pi->adapter;
1194
1195         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1196                 return;
1197         cxgbe_init_synchronized(pi);
1198         end_synchronized_op(sc, 0);
1199 }
1200
1201 static int
1202 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1203 {
1204         int rc = 0, mtu, flags, can_sleep;
1205         struct port_info *pi = ifp->if_softc;
1206         struct adapter *sc = pi->adapter;
1207         struct ifreq *ifr = (struct ifreq *)data;
1208         uint32_t mask;
1209
1210         switch (cmd) {
1211         case SIOCSIFMTU:
1212                 mtu = ifr->ifr_mtu;
1213                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1214                         return (EINVAL);
1215
1216                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1217                 if (rc)
1218                         return (rc);
1219                 ifp->if_mtu = mtu;
1220                 if (pi->flags & PORT_INIT_DONE) {
1221                         t4_update_fl_bufsize(ifp);
1222                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1223                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1224                 }
1225                 end_synchronized_op(sc, 0);
1226                 break;
1227
1228         case SIOCSIFFLAGS:
1229                 can_sleep = 0;
1230 redo_sifflags:
1231                 rc = begin_synchronized_op(sc, pi,
1232                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1233                 if (rc)
1234                         return (rc);
1235
1236                 if (ifp->if_flags & IFF_UP) {
1237                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1238                                 flags = pi->if_flags;
1239                                 if ((ifp->if_flags ^ flags) &
1240                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1241                                         if (can_sleep == 1) {
1242                                                 end_synchronized_op(sc, 0);
1243                                                 can_sleep = 0;
1244                                                 goto redo_sifflags;
1245                                         }
1246                                         rc = update_mac_settings(ifp,
1247                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1248                                 }
1249                         } else {
1250                                 if (can_sleep == 0) {
1251                                         end_synchronized_op(sc, LOCK_HELD);
1252                                         can_sleep = 1;
1253                                         goto redo_sifflags;
1254                                 }
1255                                 rc = cxgbe_init_synchronized(pi);
1256                         }
1257                         pi->if_flags = ifp->if_flags;
1258                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1259                         if (can_sleep == 0) {
1260                                 end_synchronized_op(sc, LOCK_HELD);
1261                                 can_sleep = 1;
1262                                 goto redo_sifflags;
1263                         }
1264                         rc = cxgbe_uninit_synchronized(pi);
1265                 }
1266                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1267                 break;
1268
1269         case SIOCADDMULTI:
1270         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1271                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1272                 if (rc)
1273                         return (rc);
1274                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1275                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1276                 end_synchronized_op(sc, LOCK_HELD);
1277                 break;
1278
1279         case SIOCSIFCAP:
1280                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1281                 if (rc)
1282                         return (rc);
1283
1284                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1285                 if (mask & IFCAP_TXCSUM) {
1286                         ifp->if_capenable ^= IFCAP_TXCSUM;
1287                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1288
1289                         if (IFCAP_TSO4 & ifp->if_capenable &&
1290                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1291                                 ifp->if_capenable &= ~IFCAP_TSO4;
1292                                 if_printf(ifp,
1293                                     "tso4 disabled due to -txcsum.\n");
1294                         }
1295                 }
1296                 if (mask & IFCAP_TXCSUM_IPV6) {
1297                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1298                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1299
1300                         if (IFCAP_TSO6 & ifp->if_capenable &&
1301                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1302                                 ifp->if_capenable &= ~IFCAP_TSO6;
1303                                 if_printf(ifp,
1304                                     "tso6 disabled due to -txcsum6.\n");
1305                         }
1306                 }
1307                 if (mask & IFCAP_RXCSUM)
1308                         ifp->if_capenable ^= IFCAP_RXCSUM;
1309                 if (mask & IFCAP_RXCSUM_IPV6)
1310                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1311
1312                 /*
1313                  * Note that we leave CSUM_TSO alone (it is always set).  The
1314                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1315                  * sending a TSO request our way, so it's sufficient to toggle
1316                  * IFCAP_TSOx only.
1317                  */
1318                 if (mask & IFCAP_TSO4) {
1319                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1320                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1321                                 if_printf(ifp, "enable txcsum first.\n");
1322                                 rc = EAGAIN;
1323                                 goto fail;
1324                         }
1325                         ifp->if_capenable ^= IFCAP_TSO4;
1326                 }
1327                 if (mask & IFCAP_TSO6) {
1328                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1329                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1330                                 if_printf(ifp, "enable txcsum6 first.\n");
1331                                 rc = EAGAIN;
1332                                 goto fail;
1333                         }
1334                         ifp->if_capenable ^= IFCAP_TSO6;
1335                 }
1336                 if (mask & IFCAP_LRO) {
1337 #if defined(INET) || defined(INET6)
1338                         int i;
1339                         struct sge_rxq *rxq;
1340
1341                         ifp->if_capenable ^= IFCAP_LRO;
1342                         for_each_rxq(pi, i, rxq) {
1343                                 if (ifp->if_capenable & IFCAP_LRO)
1344                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1345                                 else
1346                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1347                         }
1348 #endif
1349                 }
1350 #ifdef TCP_OFFLOAD
1351                 if (mask & IFCAP_TOE) {
1352                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1353
1354                         rc = toe_capability(pi, enable);
1355                         if (rc != 0)
1356                                 goto fail;
1357
1358                         ifp->if_capenable ^= mask;
1359                 }
1360 #endif
1361                 if (mask & IFCAP_VLAN_HWTAGGING) {
1362                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1363                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1364                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1365                 }
1366                 if (mask & IFCAP_VLAN_MTU) {
1367                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1368
1369                         /* Need to find out how to disable auto-mtu-inflation */
1370                 }
1371                 if (mask & IFCAP_VLAN_HWTSO)
1372                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1373                 if (mask & IFCAP_VLAN_HWCSUM)
1374                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1375
1376 #ifdef VLAN_CAPABILITIES
1377                 VLAN_CAPABILITIES(ifp);
1378 #endif
1379 fail:
1380                 end_synchronized_op(sc, 0);
1381                 break;
1382
1383         case SIOCSIFMEDIA:
1384         case SIOCGIFMEDIA:
1385                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1386                 break;
1387
1388         default:
1389                 rc = ether_ioctl(ifp, cmd, data);
1390         }
1391
1392         return (rc);
1393 }
1394
1395 static int
1396 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1397 {
1398         struct port_info *pi = ifp->if_softc;
1399         struct adapter *sc = pi->adapter;
1400         struct sge_txq *txq;
1401         void *items[1];
1402         int rc;
1403
1404         M_ASSERTPKTHDR(m);
1405         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1406
1407         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1408                 m_freem(m);
1409                 return (ENETDOWN);
1410         }
1411
1412         rc = parse_pkt(&m);
1413         if (__predict_false(rc != 0)) {
1414                 MPASS(m == NULL);                       /* was freed already */
1415                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1416                 return (rc);
1417         }
1418
1419         /* Select a txq. */
1420         txq = &sc->sge.txq[pi->first_txq];
1421         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1422                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1423                     pi->rsrv_noflowq);
1424
1425         items[0] = m;
1426         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1427         if (__predict_false(rc != 0))
1428                 m_freem(m);
1429
1430         return (rc);
1431 }
1432
1433 static void
1434 cxgbe_qflush(struct ifnet *ifp)
1435 {
1436         struct port_info *pi = ifp->if_softc;
1437         struct sge_txq *txq;
1438         int i;
1439
1440         /* queues do not exist if !PORT_INIT_DONE. */
1441         if (pi->flags & PORT_INIT_DONE) {
1442                 for_each_txq(pi, i, txq) {
1443                         TXQ_LOCK(txq);
1444                         txq->eq.flags &= ~EQ_ENABLED;
1445                         TXQ_UNLOCK(txq);
1446                         while (!mp_ring_is_idle(txq->r)) {
1447                                 mp_ring_check_drainage(txq->r, 0);
1448                                 pause("qflush", 1);
1449                         }
1450                 }
1451         }
1452         if_qflush(ifp);
1453 }
1454
1455 static int
1456 cxgbe_media_change(struct ifnet *ifp)
1457 {
1458         struct port_info *pi = ifp->if_softc;
1459
1460         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1461
1462         return (EOPNOTSUPP);
1463 }
1464
1465 static void
1466 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1467 {
1468         struct port_info *pi = ifp->if_softc;
1469         struct ifmedia *media = NULL;
1470         struct ifmedia_entry *cur;
1471         int speed = pi->link_cfg.speed;
1472 #ifdef INVARIANTS
1473         int data = (pi->port_type << 8) | pi->mod_type;
1474 #endif
1475
1476         if (ifp == pi->ifp)
1477                 media = &pi->media;
1478 #ifdef DEV_NETMAP
1479         else if (ifp == pi->nm_ifp)
1480                 media = &pi->nm_media;
1481 #endif
1482         MPASS(media != NULL);
1483
1484         cur = media->ifm_cur;
1485         MPASS(cur->ifm_data == data);
1486
1487         ifmr->ifm_status = IFM_AVALID;
1488         if (!pi->link_cfg.link_ok)
1489                 return;
1490
1491         ifmr->ifm_status |= IFM_ACTIVE;
1492
1493         /* active and current will differ iff current media is autoselect. */
1494         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1495                 return;
1496
1497         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1498         if (speed == SPEED_10000)
1499                 ifmr->ifm_active |= IFM_10G_T;
1500         else if (speed == SPEED_1000)
1501                 ifmr->ifm_active |= IFM_1000_T;
1502         else if (speed == SPEED_100)
1503                 ifmr->ifm_active |= IFM_100_TX;
1504         else if (speed == SPEED_10)
1505                 ifmr->ifm_active |= IFM_10_T;
1506         else
1507                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1508                             speed));
1509 }
1510
1511 void
1512 t4_fatal_err(struct adapter *sc)
1513 {
1514         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1515         t4_intr_disable(sc);
1516         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1517             device_get_nameunit(sc->dev));
1518 }
1519
1520 static int
1521 map_bars_0_and_4(struct adapter *sc)
1522 {
1523         sc->regs_rid = PCIR_BAR(0);
1524         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1525             &sc->regs_rid, RF_ACTIVE);
1526         if (sc->regs_res == NULL) {
1527                 device_printf(sc->dev, "cannot map registers.\n");
1528                 return (ENXIO);
1529         }
1530         sc->bt = rman_get_bustag(sc->regs_res);
1531         sc->bh = rman_get_bushandle(sc->regs_res);
1532         sc->mmio_len = rman_get_size(sc->regs_res);
1533         setbit(&sc->doorbells, DOORBELL_KDB);
1534
1535         sc->msix_rid = PCIR_BAR(4);
1536         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1537             &sc->msix_rid, RF_ACTIVE);
1538         if (sc->msix_res == NULL) {
1539                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1540                 return (ENXIO);
1541         }
1542
1543         return (0);
1544 }
1545
1546 static int
1547 map_bar_2(struct adapter *sc)
1548 {
1549
1550         /*
1551          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1552          * to map it if RDMA is disabled.
1553          */
1554         if (is_t4(sc) && sc->rdmacaps == 0)
1555                 return (0);
1556
1557         sc->udbs_rid = PCIR_BAR(2);
1558         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1559             &sc->udbs_rid, RF_ACTIVE);
1560         if (sc->udbs_res == NULL) {
1561                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1562                 return (ENXIO);
1563         }
1564         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1565
1566         if (is_t5(sc)) {
1567                 setbit(&sc->doorbells, DOORBELL_UDB);
1568 #if defined(__i386__) || defined(__amd64__)
1569                 if (t5_write_combine) {
1570                         int rc;
1571
1572                         /*
1573                          * Enable write combining on BAR2.  This is the
1574                          * userspace doorbell BAR and is split into 128B
1575                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1576                          * with an egress queue.  The first 64B has the doorbell
1577                          * and the second 64B can be used to submit a tx work
1578                          * request with an implicit doorbell.
1579                          */
1580
1581                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1582                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1583                         if (rc == 0) {
1584                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1585                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1586                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1587                         } else {
1588                                 device_printf(sc->dev,
1589                                     "couldn't enable write combining: %d\n",
1590                                     rc);
1591                         }
1592
1593                         t4_write_reg(sc, A_SGE_STAT_CFG,
1594                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1595                 }
1596 #endif
1597         }
1598
1599         return (0);
1600 }
1601
1602 static const struct memwin t4_memwin[] = {
1603         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1604         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1605         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1606 };
1607
1608 static const struct memwin t5_memwin[] = {
1609         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1610         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1611         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1612 };
1613
1614 static void
1615 setup_memwin(struct adapter *sc)
1616 {
1617         const struct memwin *mw;
1618         int i, n;
1619         uint32_t bar0;
1620
1621         if (is_t4(sc)) {
1622                 /*
1623                  * Read low 32b of bar0 indirectly via the hardware backdoor
1624                  * mechanism.  Works from within PCI passthrough environments
1625                  * too, where rman_get_start() can return a different value.  We
1626                  * need to program the T4 memory window decoders with the actual
1627                  * addresses that will be coming across the PCIe link.
1628                  */
1629                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1630                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1631
1632                 mw = &t4_memwin[0];
1633                 n = nitems(t4_memwin);
1634         } else {
1635                 /* T5 uses the relative offset inside the PCIe BAR */
1636                 bar0 = 0;
1637
1638                 mw = &t5_memwin[0];
1639                 n = nitems(t5_memwin);
1640         }
1641
1642         for (i = 0; i < n; i++, mw++) {
1643                 t4_write_reg(sc,
1644                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1645                     (mw->base + bar0) | V_BIR(0) |
1646                     V_WINDOW(ilog2(mw->aperture) - 10));
1647         }
1648
1649         /* flush */
1650         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1651 }
1652
1653 /*
1654  * Verify that the memory range specified by the addr/len pair is valid and lies
1655  * entirely within a single region (EDCx or MCx).
1656  */
1657 static int
1658 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1659 {
1660         uint32_t em, addr_len, maddr, mlen;
1661
1662         /* Memory can only be accessed in naturally aligned 4 byte units */
1663         if (addr & 3 || len & 3 || len == 0)
1664                 return (EINVAL);
1665
1666         /* Enabled memories */
1667         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1668         if (em & F_EDRAM0_ENABLE) {
1669                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1670                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1671                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1672                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1673                     addr + len <= maddr + mlen)
1674                         return (0);
1675         }
1676         if (em & F_EDRAM1_ENABLE) {
1677                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1678                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1679                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1680                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1681                     addr + len <= maddr + mlen)
1682                         return (0);
1683         }
1684         if (em & F_EXT_MEM_ENABLE) {
1685                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1686                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1687                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1688                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1689                     addr + len <= maddr + mlen)
1690                         return (0);
1691         }
1692         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1693                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1694                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1695                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1696                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1697                     addr + len <= maddr + mlen)
1698                         return (0);
1699         }
1700
1701         return (EFAULT);
1702 }
1703
1704 static int
1705 fwmtype_to_hwmtype(int mtype)
1706 {
1707
1708         switch (mtype) {
1709         case FW_MEMTYPE_EDC0:
1710                 return (MEM_EDC0);
1711         case FW_MEMTYPE_EDC1:
1712                 return (MEM_EDC1);
1713         case FW_MEMTYPE_EXTMEM:
1714                 return (MEM_MC0);
1715         case FW_MEMTYPE_EXTMEM1:
1716                 return (MEM_MC1);
1717         default:
1718                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1719         }
1720 }
1721
1722 /*
1723  * Verify that the memory range specified by the memtype/offset/len pair is
1724  * valid and lies entirely within the memtype specified.  The global address of
1725  * the start of the range is returned in addr.
1726  */
1727 static int
1728 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1729     uint32_t *addr)
1730 {
1731         uint32_t em, addr_len, maddr, mlen;
1732
1733         /* Memory can only be accessed in naturally aligned 4 byte units */
1734         if (off & 3 || len & 3 || len == 0)
1735                 return (EINVAL);
1736
1737         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1738         switch (fwmtype_to_hwmtype(mtype)) {
1739         case MEM_EDC0:
1740                 if (!(em & F_EDRAM0_ENABLE))
1741                         return (EINVAL);
1742                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1743                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1744                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1745                 break;
1746         case MEM_EDC1:
1747                 if (!(em & F_EDRAM1_ENABLE))
1748                         return (EINVAL);
1749                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1750                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1751                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1752                 break;
1753         case MEM_MC:
1754                 if (!(em & F_EXT_MEM_ENABLE))
1755                         return (EINVAL);
1756                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1757                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1758                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1759                 break;
1760         case MEM_MC1:
1761                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1762                         return (EINVAL);
1763                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1764                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1765                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1766                 break;
1767         default:
1768                 return (EINVAL);
1769         }
1770
1771         if (mlen > 0 && off < mlen && off + len <= mlen) {
1772                 *addr = maddr + off;    /* global address */
1773                 return (0);
1774         }
1775
1776         return (EFAULT);
1777 }
1778
1779 static void
1780 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1781 {
1782         const struct memwin *mw;
1783
1784         if (is_t4(sc)) {
1785                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1786                     ("%s: incorrect memwin# (%d)", __func__, win));
1787                 mw = &t4_memwin[win];
1788         } else {
1789                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1790                     ("%s: incorrect memwin# (%d)", __func__, win));
1791                 mw = &t5_memwin[win];
1792         }
1793
1794         if (base != NULL)
1795                 *base = mw->base;
1796         if (aperture != NULL)
1797                 *aperture = mw->aperture;
1798 }
1799
1800 /*
1801  * Positions the memory window such that it can be used to access the specified
1802  * address in the chip's address space.  The return value is the offset of addr
1803  * from the start of the window.
1804  */
1805 static uint32_t
1806 position_memwin(struct adapter *sc, int n, uint32_t addr)
1807 {
1808         uint32_t start, pf;
1809         uint32_t reg;
1810
1811         KASSERT(n >= 0 && n <= 3,
1812             ("%s: invalid window %d.", __func__, n));
1813         KASSERT((addr & 3) == 0,
1814             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1815
1816         if (is_t4(sc)) {
1817                 pf = 0;
1818                 start = addr & ~0xf;    /* start must be 16B aligned */
1819         } else {
1820                 pf = V_PFNUM(sc->pf);
1821                 start = addr & ~0x7f;   /* start must be 128B aligned */
1822         }
1823         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1824
1825         t4_write_reg(sc, reg, start | pf);
1826         t4_read_reg(sc, reg);
1827
1828         return (addr - start);
1829 }
1830
1831 static int
1832 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1833     struct intrs_and_queues *iaq)
1834 {
1835         int rc, itype, navail, nrxq10g, nrxq1g, n;
1836         int nofldrxq10g = 0, nofldrxq1g = 0;
1837         int nnmrxq10g = 0, nnmrxq1g = 0;
1838
1839         bzero(iaq, sizeof(*iaq));
1840
1841         iaq->ntxq10g = t4_ntxq10g;
1842         iaq->ntxq1g = t4_ntxq1g;
1843         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1844         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1845         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1846 #ifdef TCP_OFFLOAD
1847         if (is_offload(sc)) {
1848                 iaq->nofldtxq10g = t4_nofldtxq10g;
1849                 iaq->nofldtxq1g = t4_nofldtxq1g;
1850                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1851                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1852         }
1853 #endif
1854 #ifdef DEV_NETMAP
1855         iaq->nnmtxq10g = t4_nnmtxq10g;
1856         iaq->nnmtxq1g = t4_nnmtxq1g;
1857         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1858         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1859 #endif
1860
1861         for (itype = INTR_MSIX; itype; itype >>= 1) {
1862
1863                 if ((itype & t4_intr_types) == 0)
1864                         continue;       /* not allowed */
1865
1866                 if (itype == INTR_MSIX)
1867                         navail = pci_msix_count(sc->dev);
1868                 else if (itype == INTR_MSI)
1869                         navail = pci_msi_count(sc->dev);
1870                 else
1871                         navail = 1;
1872 restart:
1873                 if (navail == 0)
1874                         continue;
1875
1876                 iaq->intr_type = itype;
1877                 iaq->intr_flags_10g = 0;
1878                 iaq->intr_flags_1g = 0;
1879
1880                 /*
1881                  * Best option: an interrupt vector for errors, one for the
1882                  * firmware event queue, and one for every rxq (NIC, TOE, and
1883                  * netmap).
1884                  */
1885                 iaq->nirq = T4_EXTRA_INTR;
1886                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1887                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1888                 if (iaq->nirq <= navail &&
1889                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1890                         iaq->intr_flags_10g = INTR_ALL;
1891                         iaq->intr_flags_1g = INTR_ALL;
1892                         goto allocate;
1893                 }
1894
1895                 /*
1896                  * Second best option: a vector for errors, one for the firmware
1897                  * event queue, and vectors for either all the NIC rx queues or
1898                  * all the TOE rx queues.  The queues that don't get vectors
1899                  * will forward their interrupts to those that do.
1900                  *
1901                  * Note: netmap rx queues cannot be created early and so they
1902                  * can't be setup to receive forwarded interrupts for others.
1903                  */
1904                 iaq->nirq = T4_EXTRA_INTR;
1905                 if (nrxq10g >= nofldrxq10g) {
1906                         iaq->intr_flags_10g = INTR_RXQ;
1907                         iaq->nirq += n10g * nrxq10g;
1908 #ifdef DEV_NETMAP
1909                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1910 #endif
1911                 } else {
1912                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1913                         iaq->nirq += n10g * nofldrxq10g;
1914 #ifdef DEV_NETMAP
1915                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1916 #endif
1917                 }
1918                 if (nrxq1g >= nofldrxq1g) {
1919                         iaq->intr_flags_1g = INTR_RXQ;
1920                         iaq->nirq += n1g * nrxq1g;
1921 #ifdef DEV_NETMAP
1922                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1923 #endif
1924                 } else {
1925                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1926                         iaq->nirq += n1g * nofldrxq1g;
1927 #ifdef DEV_NETMAP
1928                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1929 #endif
1930                 }
1931                 if (iaq->nirq <= navail &&
1932                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1933                         goto allocate;
1934
1935                 /*
1936                  * Next best option: an interrupt vector for errors, one for the
1937                  * firmware event queue, and at least one per port.  At this
1938                  * point we know we'll have to downsize nrxq and/or nofldrxq
1939                  * and/or nnmrxq to fit what's available to us.
1940                  */
1941                 iaq->nirq = T4_EXTRA_INTR;
1942                 iaq->nirq += n10g + n1g;
1943                 if (iaq->nirq <= navail) {
1944                         int leftover = navail - iaq->nirq;
1945
1946                         if (n10g > 0) {
1947                                 int target = max(nrxq10g, nofldrxq10g);
1948
1949                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1950                                     INTR_RXQ : INTR_OFLD_RXQ;
1951
1952                                 n = 1;
1953                                 while (n < target && leftover >= n10g) {
1954                                         leftover -= n10g;
1955                                         iaq->nirq += n10g;
1956                                         n++;
1957                                 }
1958                                 iaq->nrxq10g = min(n, nrxq10g);
1959 #ifdef TCP_OFFLOAD
1960                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1961 #endif
1962 #ifdef DEV_NETMAP
1963                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1964 #endif
1965                         }
1966
1967                         if (n1g > 0) {
1968                                 int target = max(nrxq1g, nofldrxq1g);
1969
1970                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1971                                     INTR_RXQ : INTR_OFLD_RXQ;
1972
1973                                 n = 1;
1974                                 while (n < target && leftover >= n1g) {
1975                                         leftover -= n1g;
1976                                         iaq->nirq += n1g;
1977                                         n++;
1978                                 }
1979                                 iaq->nrxq1g = min(n, nrxq1g);
1980 #ifdef TCP_OFFLOAD
1981                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1982 #endif
1983 #ifdef DEV_NETMAP
1984                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1985 #endif
1986                         }
1987
1988                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1989                                 goto allocate;
1990                 }
1991
1992                 /*
1993                  * Least desirable option: one interrupt vector for everything.
1994                  */
1995                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1996                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
1997 #ifdef TCP_OFFLOAD
1998                 if (is_offload(sc))
1999                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2000 #endif
2001 #ifdef DEV_NETMAP
2002                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2003 #endif
2004
2005 allocate:
2006                 navail = iaq->nirq;
2007                 rc = 0;
2008                 if (itype == INTR_MSIX)
2009                         rc = pci_alloc_msix(sc->dev, &navail);
2010                 else if (itype == INTR_MSI)
2011                         rc = pci_alloc_msi(sc->dev, &navail);
2012
2013                 if (rc == 0) {
2014                         if (navail == iaq->nirq)
2015                                 return (0);
2016
2017                         /*
2018                          * Didn't get the number requested.  Use whatever number
2019                          * the kernel is willing to allocate (it's in navail).
2020                          */
2021                         device_printf(sc->dev, "fewer vectors than requested, "
2022                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2023                             itype, iaq->nirq, navail);
2024                         pci_release_msi(sc->dev);
2025                         goto restart;
2026                 }
2027
2028                 device_printf(sc->dev,
2029                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2030                     itype, rc, iaq->nirq, navail);
2031         }
2032
2033         device_printf(sc->dev,
2034             "failed to find a usable interrupt type.  "
2035             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2036             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2037
2038         return (ENXIO);
2039 }
2040
2041 #define FW_VERSION(chip) ( \
2042     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2043     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2044     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2045     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2046 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2047
2048 struct fw_info {
2049         uint8_t chip;
2050         char *kld_name;
2051         char *fw_mod_name;
2052         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2053 } fw_info[] = {
2054         {
2055                 .chip = CHELSIO_T4,
2056                 .kld_name = "t4fw_cfg",
2057                 .fw_mod_name = "t4fw",
2058                 .fw_hdr = {
2059                         .chip = FW_HDR_CHIP_T4,
2060                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2061                         .intfver_nic = FW_INTFVER(T4, NIC),
2062                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2063                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2064                         .intfver_ri = FW_INTFVER(T4, RI),
2065                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2066                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2067                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2068                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2069                 },
2070         }, {
2071                 .chip = CHELSIO_T5,
2072                 .kld_name = "t5fw_cfg",
2073                 .fw_mod_name = "t5fw",
2074                 .fw_hdr = {
2075                         .chip = FW_HDR_CHIP_T5,
2076                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2077                         .intfver_nic = FW_INTFVER(T5, NIC),
2078                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2079                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2080                         .intfver_ri = FW_INTFVER(T5, RI),
2081                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2082                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2083                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2084                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2085                 },
2086         }
2087 };
2088
2089 static struct fw_info *
2090 find_fw_info(int chip)
2091 {
2092         int i;
2093
2094         for (i = 0; i < nitems(fw_info); i++) {
2095                 if (fw_info[i].chip == chip)
2096                         return (&fw_info[i]);
2097         }
2098         return (NULL);
2099 }
2100
2101 /*
2102  * Is the given firmware API compatible with the one the driver was compiled
2103  * with?
2104  */
2105 static int
2106 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2107 {
2108
2109         /* short circuit if it's the exact same firmware version */
2110         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2111                 return (1);
2112
2113         /*
2114          * XXX: Is this too conservative?  Perhaps I should limit this to the
2115          * features that are supported in the driver.
2116          */
2117 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2118         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2119             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2120             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2121                 return (1);
2122 #undef SAME_INTF
2123
2124         return (0);
2125 }
2126
2127 /*
2128  * The firmware in the KLD is usable, but should it be installed?  This routine
2129  * explains itself in detail if it indicates the KLD firmware should be
2130  * installed.
2131  */
2132 static int
2133 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2134 {
2135         const char *reason;
2136
2137         if (!card_fw_usable) {
2138                 reason = "incompatible or unusable";
2139                 goto install;
2140         }
2141
2142         if (k > c) {
2143                 reason = "older than the version bundled with this driver";
2144                 goto install;
2145         }
2146
2147         if (t4_fw_install == 2 && k != c) {
2148                 reason = "different than the version bundled with this driver";
2149                 goto install;
2150         }
2151
2152         return (0);
2153
2154 install:
2155         if (t4_fw_install == 0) {
2156                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2157                     "but the driver is prohibited from installing a different "
2158                     "firmware on the card.\n",
2159                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2160                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2161
2162                 return (0);
2163         }
2164
2165         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2166             "installing firmware %u.%u.%u.%u on card.\n",
2167             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2168             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2169             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2170             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2171
2172         return (1);
2173 }
2174 /*
2175  * Establish contact with the firmware and determine if we are the master driver
2176  * or not, and whether we are responsible for chip initialization.
2177  */
2178 static int
2179 prep_firmware(struct adapter *sc)
2180 {
2181         const struct firmware *fw = NULL, *default_cfg;
2182         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2183         enum dev_state state;
2184         struct fw_info *fw_info;
2185         struct fw_hdr *card_fw;         /* fw on the card */
2186         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2187         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2188                                            against */
2189
2190         /* Contact firmware. */
2191         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2192         if (rc < 0 || state == DEV_STATE_ERR) {
2193                 rc = -rc;
2194                 device_printf(sc->dev,
2195                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2196                 return (rc);
2197         }
2198         pf = rc;
2199         if (pf == sc->mbox)
2200                 sc->flags |= MASTER_PF;
2201         else if (state == DEV_STATE_UNINIT) {
2202                 /*
2203                  * We didn't get to be the master so we definitely won't be
2204                  * configuring the chip.  It's a bug if someone else hasn't
2205                  * configured it already.
2206                  */
2207                 device_printf(sc->dev, "couldn't be master(%d), "
2208                     "device not already initialized either(%d).\n", rc, state);
2209                 return (EDOOFUS);
2210         }
2211
2212         /* This is the firmware whose headers the driver was compiled against */
2213         fw_info = find_fw_info(chip_id(sc));
2214         if (fw_info == NULL) {
2215                 device_printf(sc->dev,
2216                     "unable to look up firmware information for chip %d.\n",
2217                     chip_id(sc));
2218                 return (EINVAL);
2219         }
2220         drv_fw = &fw_info->fw_hdr;
2221
2222         /*
2223          * The firmware KLD contains many modules.  The KLD name is also the
2224          * name of the module that contains the default config file.
2225          */
2226         default_cfg = firmware_get(fw_info->kld_name);
2227
2228         /* Read the header of the firmware on the card */
2229         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2230         rc = -t4_read_flash(sc, FLASH_FW_START,
2231             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2232         if (rc == 0)
2233                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2234         else {
2235                 device_printf(sc->dev,
2236                     "Unable to read card's firmware header: %d\n", rc);
2237                 card_fw_usable = 0;
2238         }
2239
2240         /* This is the firmware in the KLD */
2241         fw = firmware_get(fw_info->fw_mod_name);
2242         if (fw != NULL) {
2243                 kld_fw = (const void *)fw->data;
2244                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2245         } else {
2246                 kld_fw = NULL;
2247                 kld_fw_usable = 0;
2248         }
2249
2250         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2251             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2252                 /*
2253                  * Common case: the firmware on the card is an exact match and
2254                  * the KLD is an exact match too, or the KLD is
2255                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2256                  * here -- use cxgbetool loadfw if you want to reinstall the
2257                  * same firmware as the one on the card.
2258                  */
2259         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2260             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2261             be32toh(card_fw->fw_ver))) {
2262
2263                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2264                 if (rc != 0) {
2265                         device_printf(sc->dev,
2266                             "failed to install firmware: %d\n", rc);
2267                         goto done;
2268                 }
2269
2270                 /* Installed successfully, update the cached header too. */
2271                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2272                 card_fw_usable = 1;
2273                 need_fw_reset = 0;      /* already reset as part of load_fw */
2274         }
2275
2276         if (!card_fw_usable) {
2277                 uint32_t d, c, k;
2278
2279                 d = ntohl(drv_fw->fw_ver);
2280                 c = ntohl(card_fw->fw_ver);
2281                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2282
2283                 device_printf(sc->dev, "Cannot find a usable firmware: "
2284                     "fw_install %d, chip state %d, "
2285                     "driver compiled with %d.%d.%d.%d, "
2286                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2287                     t4_fw_install, state,
2288                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2289                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2290                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2291                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2292                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2293                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2294                 rc = EINVAL;
2295                 goto done;
2296         }
2297
2298         /* We're using whatever's on the card and it's known to be good. */
2299         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2300         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2301             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2302             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2303             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2304             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2305         t4_get_tp_version(sc, &sc->params.tp_vers);
2306
2307         /* Reset device */
2308         if (need_fw_reset &&
2309             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2310                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2311                 if (rc != ETIMEDOUT && rc != EIO)
2312                         t4_fw_bye(sc, sc->mbox);
2313                 goto done;
2314         }
2315         sc->flags |= FW_OK;
2316
2317         rc = get_params__pre_init(sc);
2318         if (rc != 0)
2319                 goto done; /* error message displayed already */
2320
2321         /* Partition adapter resources as specified in the config file. */
2322         if (state == DEV_STATE_UNINIT) {
2323
2324                 KASSERT(sc->flags & MASTER_PF,
2325                     ("%s: trying to change chip settings when not master.",
2326                     __func__));
2327
2328                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2329                 if (rc != 0)
2330                         goto done;      /* error message displayed already */
2331
2332                 t4_tweak_chip_settings(sc);
2333
2334                 /* get basic stuff going */
2335                 rc = -t4_fw_initialize(sc, sc->mbox);
2336                 if (rc != 0) {
2337                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2338                         goto done;
2339                 }
2340         } else {
2341                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2342                 sc->cfcsum = 0;
2343         }
2344
2345 done:
2346         free(card_fw, M_CXGBE);
2347         if (fw != NULL)
2348                 firmware_put(fw, FIRMWARE_UNLOAD);
2349         if (default_cfg != NULL)
2350                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2351
2352         return (rc);
2353 }
2354
2355 #define FW_PARAM_DEV(param) \
2356         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2357          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2358 #define FW_PARAM_PFVF(param) \
2359         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2360          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2361
2362 /*
2363  * Partition chip resources for use between various PFs, VFs, etc.
2364  */
2365 static int
2366 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2367     const char *name_prefix)
2368 {
2369         const struct firmware *cfg = NULL;
2370         int rc = 0;
2371         struct fw_caps_config_cmd caps;
2372         uint32_t mtype, moff, finicsum, cfcsum;
2373
2374         /*
2375          * Figure out what configuration file to use.  Pick the default config
2376          * file for the card if the user hasn't specified one explicitly.
2377          */
2378         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2379         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2380                 /* Card specific overrides go here. */
2381                 if (pci_get_device(sc->dev) == 0x440a)
2382                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2383                 if (is_fpga(sc))
2384                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2385         }
2386
2387         /*
2388          * We need to load another module if the profile is anything except
2389          * "default" or "flash".
2390          */
2391         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2392             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2393                 char s[32];
2394
2395                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2396                 cfg = firmware_get(s);
2397                 if (cfg == NULL) {
2398                         if (default_cfg != NULL) {
2399                                 device_printf(sc->dev,
2400                                     "unable to load module \"%s\" for "
2401                                     "configuration profile \"%s\", will use "
2402                                     "the default config file instead.\n",
2403                                     s, sc->cfg_file);
2404                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2405                                     "%s", DEFAULT_CF);
2406                         } else {
2407                                 device_printf(sc->dev,
2408                                     "unable to load module \"%s\" for "
2409                                     "configuration profile \"%s\", will use "
2410                                     "the config file on the card's flash "
2411                                     "instead.\n", s, sc->cfg_file);
2412                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2413                                     "%s", FLASH_CF);
2414                         }
2415                 }
2416         }
2417
2418         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2419             default_cfg == NULL) {
2420                 device_printf(sc->dev,
2421                     "default config file not available, will use the config "
2422                     "file on the card's flash instead.\n");
2423                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2424         }
2425
2426         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2427                 u_int cflen, i, n;
2428                 const uint32_t *cfdata;
2429                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2430
2431                 KASSERT(cfg != NULL || default_cfg != NULL,
2432                     ("%s: no config to upload", __func__));
2433
2434                 /*
2435                  * Ask the firmware where it wants us to upload the config file.
2436                  */
2437                 param = FW_PARAM_DEV(CF);
2438                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2439                 if (rc != 0) {
2440                         /* No support for config file?  Shouldn't happen. */
2441                         device_printf(sc->dev,
2442                             "failed to query config file location: %d.\n", rc);
2443                         goto done;
2444                 }
2445                 mtype = G_FW_PARAMS_PARAM_Y(val);
2446                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2447
2448                 /*
2449                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2450                  * useless stuffing/comments at the end of the config file so
2451                  * it's ok to simply throw away the last remaining bytes when
2452                  * the config file is not an exact multiple of 4.  This also
2453                  * helps with the validate_mt_off_len check.
2454                  */
2455                 if (cfg != NULL) {
2456                         cflen = cfg->datasize & ~3;
2457                         cfdata = cfg->data;
2458                 } else {
2459                         cflen = default_cfg->datasize & ~3;
2460                         cfdata = default_cfg->data;
2461                 }
2462
2463                 if (cflen > FLASH_CFG_MAX_SIZE) {
2464                         device_printf(sc->dev,
2465                             "config file too long (%d, max allowed is %d).  "
2466                             "Will try to use the config on the card, if any.\n",
2467                             cflen, FLASH_CFG_MAX_SIZE);
2468                         goto use_config_on_flash;
2469                 }
2470
2471                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2472                 if (rc != 0) {
2473                         device_printf(sc->dev,
2474                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2475                             "Will try to use the config on the card, if any.\n",
2476                             __func__, mtype, moff, cflen, rc);
2477                         goto use_config_on_flash;
2478                 }
2479
2480                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2481                 while (cflen) {
2482                         off = position_memwin(sc, 2, addr);
2483                         n = min(cflen, mw_aperture - off);
2484                         for (i = 0; i < n; i += 4)
2485                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2486                         cflen -= n;
2487                         addr += n;
2488                 }
2489         } else {
2490 use_config_on_flash:
2491                 mtype = FW_MEMTYPE_FLASH;
2492                 moff = t4_flash_cfg_addr(sc);
2493         }
2494
2495         bzero(&caps, sizeof(caps));
2496         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2497             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2498         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2499             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2500             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2501         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2502         if (rc != 0) {
2503                 device_printf(sc->dev,
2504                     "failed to pre-process config file: %d "
2505                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2506                 goto done;
2507         }
2508
2509         finicsum = be32toh(caps.finicsum);
2510         cfcsum = be32toh(caps.cfcsum);
2511         if (finicsum != cfcsum) {
2512                 device_printf(sc->dev,
2513                     "WARNING: config file checksum mismatch: %08x %08x\n",
2514                     finicsum, cfcsum);
2515         }
2516         sc->cfcsum = cfcsum;
2517
2518 #define LIMIT_CAPS(x) do { \
2519         caps.x &= htobe16(t4_##x##_allowed); \
2520 } while (0)
2521
2522         /*
2523          * Let the firmware know what features will (not) be used so it can tune
2524          * things accordingly.
2525          */
2526         LIMIT_CAPS(linkcaps);
2527         LIMIT_CAPS(niccaps);
2528         LIMIT_CAPS(toecaps);
2529         LIMIT_CAPS(rdmacaps);
2530         LIMIT_CAPS(iscsicaps);
2531         LIMIT_CAPS(fcoecaps);
2532 #undef LIMIT_CAPS
2533
2534         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2535             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2536         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2537         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2538         if (rc != 0) {
2539                 device_printf(sc->dev,
2540                     "failed to process config file: %d.\n", rc);
2541         }
2542 done:
2543         if (cfg != NULL)
2544                 firmware_put(cfg, FIRMWARE_UNLOAD);
2545         return (rc);
2546 }
2547
2548 /*
2549  * Retrieve parameters that are needed (or nice to have) very early.
2550  */
2551 static int
2552 get_params__pre_init(struct adapter *sc)
2553 {
2554         int rc;
2555         uint32_t param[2], val[2];
2556         struct fw_devlog_cmd cmd;
2557         struct devlog_params *dlog = &sc->params.devlog;
2558
2559         param[0] = FW_PARAM_DEV(PORTVEC);
2560         param[1] = FW_PARAM_DEV(CCLK);
2561         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2562         if (rc != 0) {
2563                 device_printf(sc->dev,
2564                     "failed to query parameters (pre_init): %d.\n", rc);
2565                 return (rc);
2566         }
2567
2568         sc->params.portvec = val[0];
2569         sc->params.nports = bitcount32(val[0]);
2570         sc->params.vpd.cclk = val[1];
2571
2572         /* Read device log parameters. */
2573         bzero(&cmd, sizeof(cmd));
2574         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2575             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2576         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2577         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2578         if (rc != 0) {
2579                 device_printf(sc->dev,
2580                     "failed to get devlog parameters: %d.\n", rc);
2581                 bzero(dlog, sizeof (*dlog));
2582                 rc = 0; /* devlog isn't critical for device operation */
2583         } else {
2584                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2585                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2586                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2587                 dlog->size = be32toh(cmd.memsize_devlog);
2588         }
2589
2590         return (rc);
2591 }
2592
2593 /*
2594  * Retrieve various parameters that are of interest to the driver.  The device
2595  * has been initialized by the firmware at this point.
2596  */
2597 static int
2598 get_params__post_init(struct adapter *sc)
2599 {
2600         int rc;
2601         uint32_t param[7], val[7];
2602         struct fw_caps_config_cmd caps;
2603
2604         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2605         param[1] = FW_PARAM_PFVF(EQ_START);
2606         param[2] = FW_PARAM_PFVF(FILTER_START);
2607         param[3] = FW_PARAM_PFVF(FILTER_END);
2608         param[4] = FW_PARAM_PFVF(L2T_START);
2609         param[5] = FW_PARAM_PFVF(L2T_END);
2610         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2611         if (rc != 0) {
2612                 device_printf(sc->dev,
2613                     "failed to query parameters (post_init): %d.\n", rc);
2614                 return (rc);
2615         }
2616
2617         sc->sge.iq_start = val[0];
2618         sc->sge.eq_start = val[1];
2619         sc->tids.ftid_base = val[2];
2620         sc->tids.nftids = val[3] - val[2] + 1;
2621         sc->params.ftid_min = val[2];
2622         sc->params.ftid_max = val[3];
2623         sc->vres.l2t.start = val[4];
2624         sc->vres.l2t.size = val[5] - val[4] + 1;
2625         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2626             ("%s: L2 table size (%u) larger than expected (%u)",
2627             __func__, sc->vres.l2t.size, L2T_SIZE));
2628
2629         /* get capabilites */
2630         bzero(&caps, sizeof(caps));
2631         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2632             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2633         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2634         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2635         if (rc != 0) {
2636                 device_printf(sc->dev,
2637                     "failed to get card capabilities: %d.\n", rc);
2638                 return (rc);
2639         }
2640
2641 #define READ_CAPS(x) do { \
2642         sc->x = htobe16(caps.x); \
2643 } while (0)
2644         READ_CAPS(linkcaps);
2645         READ_CAPS(niccaps);
2646         READ_CAPS(toecaps);
2647         READ_CAPS(rdmacaps);
2648         READ_CAPS(iscsicaps);
2649         READ_CAPS(fcoecaps);
2650
2651         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2652                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2653                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2654                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2655                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2656                 if (rc != 0) {
2657                         device_printf(sc->dev,
2658                             "failed to query NIC parameters: %d.\n", rc);
2659                         return (rc);
2660                 }
2661                 sc->tids.etid_base = val[0];
2662                 sc->params.etid_min = val[0];
2663                 sc->tids.netids = val[1] - val[0] + 1;
2664                 sc->params.netids = sc->tids.netids;
2665                 sc->params.eo_wr_cred = val[2];
2666                 sc->params.ethoffload = 1;
2667         }
2668
2669         if (sc->toecaps) {
2670                 /* query offload-related parameters */
2671                 param[0] = FW_PARAM_DEV(NTID);
2672                 param[1] = FW_PARAM_PFVF(SERVER_START);
2673                 param[2] = FW_PARAM_PFVF(SERVER_END);
2674                 param[3] = FW_PARAM_PFVF(TDDP_START);
2675                 param[4] = FW_PARAM_PFVF(TDDP_END);
2676                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2677                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2678                 if (rc != 0) {
2679                         device_printf(sc->dev,
2680                             "failed to query TOE parameters: %d.\n", rc);
2681                         return (rc);
2682                 }
2683                 sc->tids.ntids = val[0];
2684                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2685                 sc->tids.stid_base = val[1];
2686                 sc->tids.nstids = val[2] - val[1] + 1;
2687                 sc->vres.ddp.start = val[3];
2688                 sc->vres.ddp.size = val[4] - val[3] + 1;
2689                 sc->params.ofldq_wr_cred = val[5];
2690                 sc->params.offload = 1;
2691         }
2692         if (sc->rdmacaps) {
2693                 param[0] = FW_PARAM_PFVF(STAG_START);
2694                 param[1] = FW_PARAM_PFVF(STAG_END);
2695                 param[2] = FW_PARAM_PFVF(RQ_START);
2696                 param[3] = FW_PARAM_PFVF(RQ_END);
2697                 param[4] = FW_PARAM_PFVF(PBL_START);
2698                 param[5] = FW_PARAM_PFVF(PBL_END);
2699                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2700                 if (rc != 0) {
2701                         device_printf(sc->dev,
2702                             "failed to query RDMA parameters(1): %d.\n", rc);
2703                         return (rc);
2704                 }
2705                 sc->vres.stag.start = val[0];
2706                 sc->vres.stag.size = val[1] - val[0] + 1;
2707                 sc->vres.rq.start = val[2];
2708                 sc->vres.rq.size = val[3] - val[2] + 1;
2709                 sc->vres.pbl.start = val[4];
2710                 sc->vres.pbl.size = val[5] - val[4] + 1;
2711
2712                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2713                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2714                 param[2] = FW_PARAM_PFVF(CQ_START);
2715                 param[3] = FW_PARAM_PFVF(CQ_END);
2716                 param[4] = FW_PARAM_PFVF(OCQ_START);
2717                 param[5] = FW_PARAM_PFVF(OCQ_END);
2718                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2719                 if (rc != 0) {
2720                         device_printf(sc->dev,
2721                             "failed to query RDMA parameters(2): %d.\n", rc);
2722                         return (rc);
2723                 }
2724                 sc->vres.qp.start = val[0];
2725                 sc->vres.qp.size = val[1] - val[0] + 1;
2726                 sc->vres.cq.start = val[2];
2727                 sc->vres.cq.size = val[3] - val[2] + 1;
2728                 sc->vres.ocq.start = val[4];
2729                 sc->vres.ocq.size = val[5] - val[4] + 1;
2730         }
2731         if (sc->iscsicaps) {
2732                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2733                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2734                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2735                 if (rc != 0) {
2736                         device_printf(sc->dev,
2737                             "failed to query iSCSI parameters: %d.\n", rc);
2738                         return (rc);
2739                 }
2740                 sc->vres.iscsi.start = val[0];
2741                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2742         }
2743
2744         /*
2745          * We've got the params we wanted to query via the firmware.  Now grab
2746          * some others directly from the chip.
2747          */
2748         rc = t4_read_chip_settings(sc);
2749
2750         return (rc);
2751 }
2752
2753 static int
2754 set_params__post_init(struct adapter *sc)
2755 {
2756         uint32_t param, val;
2757
2758         /* ask for encapsulated CPLs */
2759         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2760         val = 1;
2761         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2762
2763         return (0);
2764 }
2765
2766 #undef FW_PARAM_PFVF
2767 #undef FW_PARAM_DEV
2768
2769 static void
2770 t4_set_desc(struct adapter *sc)
2771 {
2772         char buf[128];
2773         struct adapter_params *p = &sc->params;
2774
2775         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2776             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2777             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2778
2779         device_set_desc_copy(sc->dev, buf);
2780 }
2781
2782 static void
2783 build_medialist(struct port_info *pi, struct ifmedia *media)
2784 {
2785         int data, m;
2786
2787         PORT_LOCK(pi);
2788
2789         ifmedia_removeall(media);
2790
2791         m = IFM_ETHER | IFM_FDX;
2792         data = (pi->port_type << 8) | pi->mod_type;
2793
2794         switch(pi->port_type) {
2795         case FW_PORT_TYPE_BT_XFI:
2796         case FW_PORT_TYPE_BT_XAUI:
2797                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2798                 /* fall through */
2799
2800         case FW_PORT_TYPE_BT_SGMII:
2801                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2802                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2803                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2804                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2805                 break;
2806
2807         case FW_PORT_TYPE_CX4:
2808                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2809                 ifmedia_set(media, m | IFM_10G_CX4);
2810                 break;
2811
2812         case FW_PORT_TYPE_QSFP_10G:
2813         case FW_PORT_TYPE_SFP:
2814         case FW_PORT_TYPE_FIBER_XFI:
2815         case FW_PORT_TYPE_FIBER_XAUI:
2816                 switch (pi->mod_type) {
2817
2818                 case FW_PORT_MOD_TYPE_LR:
2819                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2820                         ifmedia_set(media, m | IFM_10G_LR);
2821                         break;
2822
2823                 case FW_PORT_MOD_TYPE_SR:
2824                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2825                         ifmedia_set(media, m | IFM_10G_SR);
2826                         break;
2827
2828                 case FW_PORT_MOD_TYPE_LRM:
2829                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2830                         ifmedia_set(media, m | IFM_10G_LRM);
2831                         break;
2832
2833                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2834                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2835                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2836                         ifmedia_set(media, m | IFM_10G_TWINAX);
2837                         break;
2838
2839                 case FW_PORT_MOD_TYPE_NONE:
2840                         m &= ~IFM_FDX;
2841                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2842                         ifmedia_set(media, m | IFM_NONE);
2843                         break;
2844
2845                 case FW_PORT_MOD_TYPE_NA:
2846                 case FW_PORT_MOD_TYPE_ER:
2847                 default:
2848                         device_printf(pi->dev,
2849                             "unknown port_type (%d), mod_type (%d)\n",
2850                             pi->port_type, pi->mod_type);
2851                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2852                         ifmedia_set(media, m | IFM_UNKNOWN);
2853                         break;
2854                 }
2855                 break;
2856
2857         case FW_PORT_TYPE_QSFP:
2858                 switch (pi->mod_type) {
2859
2860                 case FW_PORT_MOD_TYPE_LR:
2861                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2862                         ifmedia_set(media, m | IFM_40G_LR4);
2863                         break;
2864
2865                 case FW_PORT_MOD_TYPE_SR:
2866                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2867                         ifmedia_set(media, m | IFM_40G_SR4);
2868                         break;
2869
2870                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2871                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2872                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2873                         ifmedia_set(media, m | IFM_40G_CR4);
2874                         break;
2875
2876                 case FW_PORT_MOD_TYPE_NONE:
2877                         m &= ~IFM_FDX;
2878                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2879                         ifmedia_set(media, m | IFM_NONE);
2880                         break;
2881
2882                 default:
2883                         device_printf(pi->dev,
2884                             "unknown port_type (%d), mod_type (%d)\n",
2885                             pi->port_type, pi->mod_type);
2886                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2887                         ifmedia_set(media, m | IFM_UNKNOWN);
2888                         break;
2889                 }
2890                 break;
2891
2892         default:
2893                 device_printf(pi->dev,
2894                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2895                     pi->mod_type);
2896                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2897                 ifmedia_set(media, m | IFM_UNKNOWN);
2898                 break;
2899         }
2900
2901         PORT_UNLOCK(pi);
2902 }
2903
2904 #define FW_MAC_EXACT_CHUNK      7
2905
2906 /*
2907  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2908  * indicates which parameters should be programmed (the rest are left alone).
2909  */
2910 int
2911 update_mac_settings(struct ifnet *ifp, int flags)
2912 {
2913         int rc = 0;
2914         struct port_info *pi = ifp->if_softc;
2915         struct adapter *sc = pi->adapter;
2916         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2917         uint16_t viid = 0xffff;
2918         int16_t *xact_addr_filt = NULL;
2919
2920         ASSERT_SYNCHRONIZED_OP(sc);
2921         KASSERT(flags, ("%s: not told what to update.", __func__));
2922
2923         if (ifp == pi->ifp) {
2924                 viid = pi->viid;
2925                 xact_addr_filt = &pi->xact_addr_filt;
2926         }
2927 #ifdef DEV_NETMAP
2928         else if (ifp == pi->nm_ifp) {
2929                 viid = pi->nm_viid;
2930                 xact_addr_filt = &pi->nm_xact_addr_filt;
2931         }
2932 #endif
2933         if (flags & XGMAC_MTU)
2934                 mtu = ifp->if_mtu;
2935
2936         if (flags & XGMAC_PROMISC)
2937                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2938
2939         if (flags & XGMAC_ALLMULTI)
2940                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2941
2942         if (flags & XGMAC_VLANEX)
2943                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2944
2945         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2946                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2947                     1, vlanex, false);
2948                 if (rc) {
2949                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2950                             rc);
2951                         return (rc);
2952                 }
2953         }
2954
2955         if (flags & XGMAC_UCADDR) {
2956                 uint8_t ucaddr[ETHER_ADDR_LEN];
2957
2958                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2959                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2960                     true, true);
2961                 if (rc < 0) {
2962                         rc = -rc;
2963                         if_printf(ifp, "change_mac failed: %d\n", rc);
2964                         return (rc);
2965                 } else {
2966                         *xact_addr_filt = rc;
2967                         rc = 0;
2968                 }
2969         }
2970
2971         if (flags & XGMAC_MCADDRS) {
2972                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2973                 int del = 1;
2974                 uint64_t hash = 0;
2975                 struct ifmultiaddr *ifma;
2976                 int i = 0, j;
2977
2978                 if_maddr_rlock(ifp);
2979                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2980                         if (ifma->ifma_addr->sa_family != AF_LINK)
2981                                 continue;
2982                         mcaddr[i] =
2983                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2984                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
2985                         i++;
2986
2987                         if (i == FW_MAC_EXACT_CHUNK) {
2988                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2989                                     i, mcaddr, NULL, &hash, 0);
2990                                 if (rc < 0) {
2991                                         rc = -rc;
2992                                         for (j = 0; j < i; j++) {
2993                                                 if_printf(ifp,
2994                                                     "failed to add mc address"
2995                                                     " %02x:%02x:%02x:"
2996                                                     "%02x:%02x:%02x rc=%d\n",
2997                                                     mcaddr[j][0], mcaddr[j][1],
2998                                                     mcaddr[j][2], mcaddr[j][3],
2999                                                     mcaddr[j][4], mcaddr[j][5],
3000                                                     rc);
3001                                         }
3002                                         goto mcfail;
3003                                 }
3004                                 del = 0;
3005                                 i = 0;
3006                         }
3007                 }
3008                 if (i > 0) {
3009                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3010                             mcaddr, NULL, &hash, 0);
3011                         if (rc < 0) {
3012                                 rc = -rc;
3013                                 for (j = 0; j < i; j++) {
3014                                         if_printf(ifp,
3015                                             "failed to add mc address"
3016                                             " %02x:%02x:%02x:"
3017                                             "%02x:%02x:%02x rc=%d\n",
3018                                             mcaddr[j][0], mcaddr[j][1],
3019                                             mcaddr[j][2], mcaddr[j][3],
3020                                             mcaddr[j][4], mcaddr[j][5],
3021                                             rc);
3022                                 }
3023                                 goto mcfail;
3024                         }
3025                 }
3026
3027                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3028                 if (rc != 0)
3029                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3030 mcfail:
3031                 if_maddr_runlock(ifp);
3032         }
3033
3034         return (rc);
3035 }
3036
3037 /*
3038  * {begin|end}_synchronized_op must be called from the same thread.
3039  */
3040 int
3041 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3042     char *wmesg)
3043 {
3044         int rc, pri;
3045
3046 #ifdef WITNESS
3047         /* the caller thinks it's ok to sleep, but is it really? */
3048         if (flags & SLEEP_OK)
3049                 pause("t4slptst", 1);
3050 #endif
3051
3052         if (INTR_OK)
3053                 pri = PCATCH;
3054         else
3055                 pri = 0;
3056
3057         ADAPTER_LOCK(sc);
3058         for (;;) {
3059
3060                 if (pi && IS_DOOMED(pi)) {
3061                         rc = ENXIO;
3062                         goto done;
3063                 }
3064
3065                 if (!IS_BUSY(sc)) {
3066                         rc = 0;
3067                         break;
3068                 }
3069
3070                 if (!(flags & SLEEP_OK)) {
3071                         rc = EBUSY;
3072                         goto done;
3073                 }
3074
3075                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3076                         rc = EINTR;
3077                         goto done;
3078                 }
3079         }
3080
3081         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3082         SET_BUSY(sc);
3083 #ifdef INVARIANTS
3084         sc->last_op = wmesg;
3085         sc->last_op_thr = curthread;
3086 #endif
3087
3088 done:
3089         if (!(flags & HOLD_LOCK) || rc)
3090                 ADAPTER_UNLOCK(sc);
3091
3092         return (rc);
3093 }
3094
3095 /*
3096  * {begin|end}_synchronized_op must be called from the same thread.
3097  */
3098 void
3099 end_synchronized_op(struct adapter *sc, int flags)
3100 {
3101
3102         if (flags & LOCK_HELD)
3103                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3104         else
3105                 ADAPTER_LOCK(sc);
3106
3107         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3108         CLR_BUSY(sc);
3109         wakeup(&sc->flags);
3110         ADAPTER_UNLOCK(sc);
3111 }
3112
3113 static int
3114 cxgbe_init_synchronized(struct port_info *pi)
3115 {
3116         struct adapter *sc = pi->adapter;
3117         struct ifnet *ifp = pi->ifp;
3118         int rc = 0, i;
3119         struct sge_txq *txq;
3120
3121         ASSERT_SYNCHRONIZED_OP(sc);
3122
3123         if (isset(&sc->open_device_map, pi->port_id)) {
3124                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3125                     ("mismatch between open_device_map and if_drv_flags"));
3126                 return (0);     /* already running */
3127         }
3128
3129         if (!(sc->flags & FULL_INIT_DONE) &&
3130             ((rc = adapter_full_init(sc)) != 0))
3131                 return (rc);    /* error message displayed already */
3132
3133         if (!(pi->flags & PORT_INIT_DONE) &&
3134             ((rc = port_full_init(pi)) != 0))
3135                 return (rc); /* error message displayed already */
3136
3137         rc = update_mac_settings(ifp, XGMAC_ALL);
3138         if (rc)
3139                 goto done;      /* error message displayed already */
3140
3141         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3142         if (rc != 0) {
3143                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3144                 goto done;
3145         }
3146
3147         /*
3148          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3149          * if this changes.
3150          */
3151
3152         for_each_txq(pi, i, txq) {
3153                 TXQ_LOCK(txq);
3154                 txq->eq.flags |= EQ_ENABLED;
3155                 TXQ_UNLOCK(txq);
3156         }
3157
3158         /*
3159          * The first iq of the first port to come up is used for tracing.
3160          */
3161         if (sc->traceq < 0) {
3162                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3163                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3164                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3165                     V_QUEUENUMBER(sc->traceq));
3166                 pi->flags |= HAS_TRACEQ;
3167         }
3168
3169         /* all ok */
3170         setbit(&sc->open_device_map, pi->port_id);
3171         PORT_LOCK(pi);
3172         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3173         PORT_UNLOCK(pi);
3174
3175         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3176 done:
3177         if (rc != 0)
3178                 cxgbe_uninit_synchronized(pi);
3179
3180         return (rc);
3181 }
3182
3183 /*
3184  * Idempotent.
3185  */
3186 static int
3187 cxgbe_uninit_synchronized(struct port_info *pi)
3188 {
3189         struct adapter *sc = pi->adapter;
3190         struct ifnet *ifp = pi->ifp;
3191         int rc, i;
3192         struct sge_txq *txq;
3193
3194         ASSERT_SYNCHRONIZED_OP(sc);
3195
3196         /*
3197          * Disable the VI so that all its data in either direction is discarded
3198          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3199          * tick) intact as the TP can deliver negative advice or data that it's
3200          * holding in its RAM (for an offloaded connection) even after the VI is
3201          * disabled.
3202          */
3203         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3204         if (rc) {
3205                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3206                 return (rc);
3207         }
3208
3209         for_each_txq(pi, i, txq) {
3210                 TXQ_LOCK(txq);
3211                 txq->eq.flags &= ~EQ_ENABLED;
3212                 TXQ_UNLOCK(txq);
3213         }
3214
3215         clrbit(&sc->open_device_map, pi->port_id);
3216         PORT_LOCK(pi);
3217         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3218         PORT_UNLOCK(pi);
3219
3220         pi->link_cfg.link_ok = 0;
3221         pi->link_cfg.speed = 0;
3222         pi->linkdnrc = -1;
3223         t4_os_link_changed(sc, pi->port_id, 0, -1);
3224
3225         return (0);
3226 }
3227
3228 /*
3229  * It is ok for this function to fail midway and return right away.  t4_detach
3230  * will walk the entire sc->irq list and clean up whatever is valid.
3231  */
3232 static int
3233 setup_intr_handlers(struct adapter *sc)
3234 {
3235         int rc, rid, p, q;
3236         char s[8];
3237         struct irq *irq;
3238         struct port_info *pi;
3239         struct sge_rxq *rxq;
3240 #ifdef TCP_OFFLOAD
3241         struct sge_ofld_rxq *ofld_rxq;
3242 #endif
3243 #ifdef DEV_NETMAP
3244         struct sge_nm_rxq *nm_rxq;
3245 #endif
3246
3247         /*
3248          * Setup interrupts.
3249          */
3250         irq = &sc->irq[0];
3251         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3252         if (sc->intr_count == 1)
3253                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3254
3255         /* Multiple interrupts. */
3256         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3257             ("%s: too few intr.", __func__));
3258
3259         /* The first one is always error intr */
3260         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3261         if (rc != 0)
3262                 return (rc);
3263         irq++;
3264         rid++;
3265
3266         /* The second one is always the firmware event queue */
3267         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3268         if (rc != 0)
3269                 return (rc);
3270         irq++;
3271         rid++;
3272
3273         for_each_port(sc, p) {
3274                 pi = sc->port[p];
3275
3276                 if (pi->flags & INTR_RXQ) {
3277                         for_each_rxq(pi, q, rxq) {
3278                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3279                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3280                                     s);
3281                                 if (rc != 0)
3282                                         return (rc);
3283                                 irq++;
3284                                 rid++;
3285                         }
3286                 }
3287 #ifdef TCP_OFFLOAD
3288                 if (pi->flags & INTR_OFLD_RXQ) {
3289                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3290                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3291                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3292                                     ofld_rxq, s);
3293                                 if (rc != 0)
3294                                         return (rc);
3295                                 irq++;
3296                                 rid++;
3297                         }
3298                 }
3299 #endif
3300 #ifdef DEV_NETMAP
3301                 if (pi->flags & INTR_NM_RXQ) {
3302                         for_each_nm_rxq(pi, q, nm_rxq) {
3303                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3304                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3305                                     nm_rxq, s);
3306                                 if (rc != 0)
3307                                         return (rc);
3308                                 irq++;
3309                                 rid++;
3310                         }
3311                 }
3312 #endif
3313         }
3314         MPASS(irq == &sc->irq[sc->intr_count]);
3315
3316         return (0);
3317 }
3318
3319 int
3320 adapter_full_init(struct adapter *sc)
3321 {
3322         int rc, i;
3323
3324         ASSERT_SYNCHRONIZED_OP(sc);
3325         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3326         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3327             ("%s: FULL_INIT_DONE already", __func__));
3328
3329         /*
3330          * queues that belong to the adapter (not any particular port).
3331          */
3332         rc = t4_setup_adapter_queues(sc);
3333         if (rc != 0)
3334                 goto done;
3335
3336         for (i = 0; i < nitems(sc->tq); i++) {
3337                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3338                     taskqueue_thread_enqueue, &sc->tq[i]);
3339                 if (sc->tq[i] == NULL) {
3340                         device_printf(sc->dev,
3341                             "failed to allocate task queue %d\n", i);
3342                         rc = ENOMEM;
3343                         goto done;
3344                 }
3345                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3346                     device_get_nameunit(sc->dev), i);
3347         }
3348
3349         t4_intr_enable(sc);
3350         sc->flags |= FULL_INIT_DONE;
3351 done:
3352         if (rc != 0)
3353                 adapter_full_uninit(sc);
3354
3355         return (rc);
3356 }
3357
3358 int
3359 adapter_full_uninit(struct adapter *sc)
3360 {
3361         int i;
3362
3363         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3364
3365         t4_teardown_adapter_queues(sc);
3366
3367         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3368                 taskqueue_free(sc->tq[i]);
3369                 sc->tq[i] = NULL;
3370         }
3371
3372         sc->flags &= ~FULL_INIT_DONE;
3373
3374         return (0);
3375 }
3376
3377 int
3378 port_full_init(struct port_info *pi)
3379 {
3380         struct adapter *sc = pi->adapter;
3381         struct ifnet *ifp = pi->ifp;
3382         uint16_t *rss;
3383         struct sge_rxq *rxq;
3384         int rc, i, j;
3385
3386         ASSERT_SYNCHRONIZED_OP(sc);
3387         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3388             ("%s: PORT_INIT_DONE already", __func__));
3389
3390         sysctl_ctx_init(&pi->ctx);
3391         pi->flags |= PORT_SYSCTL_CTX;
3392
3393         /*
3394          * Allocate tx/rx/fl queues for this port.
3395          */
3396         rc = t4_setup_port_queues(pi);
3397         if (rc != 0)
3398                 goto done;      /* error message displayed already */
3399
3400         /*
3401          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3402          */
3403         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3404         for (i = 0; i < pi->rss_size;) {
3405                 for_each_rxq(pi, j, rxq) {
3406                         rss[i++] = rxq->iq.abs_id;
3407                         if (i == pi->rss_size)
3408                                 break;
3409                 }
3410         }
3411
3412         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3413             pi->rss_size);
3414         if (rc != 0) {
3415                 if_printf(ifp, "rss_config failed: %d\n", rc);
3416                 goto done;
3417         }
3418
3419         pi->rss = rss;
3420         pi->flags |= PORT_INIT_DONE;
3421 done:
3422         if (rc != 0)
3423                 port_full_uninit(pi);
3424
3425         return (rc);
3426 }
3427
3428 /*
3429  * Idempotent.
3430  */
3431 int
3432 port_full_uninit(struct port_info *pi)
3433 {
3434         struct adapter *sc = pi->adapter;
3435         int i;
3436         struct sge_rxq *rxq;
3437         struct sge_txq *txq;
3438 #ifdef TCP_OFFLOAD
3439         struct sge_ofld_rxq *ofld_rxq;
3440         struct sge_wrq *ofld_txq;
3441 #endif
3442
3443         if (pi->flags & PORT_INIT_DONE) {
3444
3445                 /* Need to quiesce queues.  */
3446
3447                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3448
3449                 for_each_txq(pi, i, txq) {
3450                         quiesce_txq(sc, txq);
3451                 }
3452
3453 #ifdef TCP_OFFLOAD
3454                 for_each_ofld_txq(pi, i, ofld_txq) {
3455                         quiesce_wrq(sc, ofld_txq);
3456                 }
3457 #endif
3458
3459                 for_each_rxq(pi, i, rxq) {
3460                         quiesce_iq(sc, &rxq->iq);
3461                         quiesce_fl(sc, &rxq->fl);
3462                 }
3463
3464 #ifdef TCP_OFFLOAD
3465                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3466                         quiesce_iq(sc, &ofld_rxq->iq);
3467                         quiesce_fl(sc, &ofld_rxq->fl);
3468                 }
3469 #endif
3470                 free(pi->rss, M_CXGBE);
3471         }
3472
3473         t4_teardown_port_queues(pi);
3474         pi->flags &= ~PORT_INIT_DONE;
3475
3476         return (0);
3477 }
3478
3479 static void
3480 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3481 {
3482         struct sge_eq *eq = &txq->eq;
3483         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3484
3485         (void) sc;      /* unused */
3486
3487 #ifdef INVARIANTS
3488         TXQ_LOCK(txq);
3489         MPASS((eq->flags & EQ_ENABLED) == 0);
3490         TXQ_UNLOCK(txq);
3491 #endif
3492
3493         /* Wait for the mp_ring to empty. */
3494         while (!mp_ring_is_idle(txq->r)) {
3495                 mp_ring_check_drainage(txq->r, 0);
3496                 pause("rquiesce", 1);
3497         }
3498
3499         /* Then wait for the hardware to finish. */
3500         while (spg->cidx != htobe16(eq->pidx))
3501                 pause("equiesce", 1);
3502
3503         /* Finally, wait for the driver to reclaim all descriptors. */
3504         while (eq->cidx != eq->pidx)
3505                 pause("dquiesce", 1);
3506 }
3507
3508 static void
3509 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3510 {
3511
3512         /* XXXTX */
3513 }
3514
3515 static void
3516 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3517 {
3518         (void) sc;      /* unused */
3519
3520         /* Synchronize with the interrupt handler */
3521         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3522                 pause("iqfree", 1);
3523 }
3524
3525 static void
3526 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3527 {
3528         mtx_lock(&sc->sfl_lock);
3529         FL_LOCK(fl);
3530         fl->flags |= FL_DOOMED;
3531         FL_UNLOCK(fl);
3532         mtx_unlock(&sc->sfl_lock);
3533
3534         callout_drain(&sc->sfl_callout);
3535         KASSERT((fl->flags & FL_STARVING) == 0,
3536             ("%s: still starving", __func__));
3537 }
3538
3539 static int
3540 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3541     driver_intr_t *handler, void *arg, char *name)
3542 {
3543         int rc;
3544
3545         irq->rid = rid;
3546         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3547             RF_SHAREABLE | RF_ACTIVE);
3548         if (irq->res == NULL) {
3549                 device_printf(sc->dev,
3550                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3551                 return (ENOMEM);
3552         }
3553
3554         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3555             NULL, handler, arg, &irq->tag);
3556         if (rc != 0) {
3557                 device_printf(sc->dev,
3558                     "failed to setup interrupt for rid %d, name %s: %d\n",
3559                     rid, name, rc);
3560         } else if (name)
3561                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3562
3563         return (rc);
3564 }
3565
3566 static int
3567 t4_free_irq(struct adapter *sc, struct irq *irq)
3568 {
3569         if (irq->tag)
3570                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3571         if (irq->res)
3572                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3573
3574         bzero(irq, sizeof(*irq));
3575
3576         return (0);
3577 }
3578
3579 static void
3580 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3581     unsigned int end)
3582 {
3583         uint32_t *p = (uint32_t *)(buf + start);
3584
3585         for ( ; start <= end; start += sizeof(uint32_t))
3586                 *p++ = t4_read_reg(sc, start);
3587 }
3588
3589 static void
3590 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3591 {
3592         int i, n;
3593         const unsigned int *reg_ranges;
3594         static const unsigned int t4_reg_ranges[] = {
3595                 0x1008, 0x1108,
3596                 0x1180, 0x11b4,
3597                 0x11fc, 0x123c,
3598                 0x1300, 0x173c,
3599                 0x1800, 0x18fc,
3600                 0x3000, 0x30d8,
3601                 0x30e0, 0x5924,
3602                 0x5960, 0x59d4,
3603                 0x5a00, 0x5af8,
3604                 0x6000, 0x6098,
3605                 0x6100, 0x6150,
3606                 0x6200, 0x6208,
3607                 0x6240, 0x6248,
3608                 0x6280, 0x6338,
3609                 0x6370, 0x638c,
3610                 0x6400, 0x643c,
3611                 0x6500, 0x6524,
3612                 0x6a00, 0x6a38,
3613                 0x6a60, 0x6a78,
3614                 0x6b00, 0x6b84,
3615                 0x6bf0, 0x6c84,
3616                 0x6cf0, 0x6d84,
3617                 0x6df0, 0x6e84,
3618                 0x6ef0, 0x6f84,
3619                 0x6ff0, 0x7084,
3620                 0x70f0, 0x7184,
3621                 0x71f0, 0x7284,
3622                 0x72f0, 0x7384,
3623                 0x73f0, 0x7450,
3624                 0x7500, 0x7530,
3625                 0x7600, 0x761c,
3626                 0x7680, 0x76cc,
3627                 0x7700, 0x7798,
3628                 0x77c0, 0x77fc,
3629                 0x7900, 0x79fc,
3630                 0x7b00, 0x7c38,
3631                 0x7d00, 0x7efc,
3632                 0x8dc0, 0x8e1c,
3633                 0x8e30, 0x8e78,
3634                 0x8ea0, 0x8f6c,
3635                 0x8fc0, 0x9074,
3636                 0x90fc, 0x90fc,
3637                 0x9400, 0x9458,
3638                 0x9600, 0x96bc,
3639                 0x9800, 0x9808,
3640                 0x9820, 0x983c,
3641                 0x9850, 0x9864,
3642                 0x9c00, 0x9c6c,
3643                 0x9c80, 0x9cec,
3644                 0x9d00, 0x9d6c,
3645                 0x9d80, 0x9dec,
3646                 0x9e00, 0x9e6c,
3647                 0x9e80, 0x9eec,
3648                 0x9f00, 0x9f6c,
3649                 0x9f80, 0x9fec,
3650                 0xd004, 0xd03c,
3651                 0xdfc0, 0xdfe0,
3652                 0xe000, 0xea7c,
3653                 0xf000, 0x11110,
3654                 0x11118, 0x11190,
3655                 0x19040, 0x1906c,
3656                 0x19078, 0x19080,
3657                 0x1908c, 0x19124,
3658                 0x19150, 0x191b0,
3659                 0x191d0, 0x191e8,
3660                 0x19238, 0x1924c,
3661                 0x193f8, 0x19474,
3662                 0x19490, 0x194f8,
3663                 0x19800, 0x19f30,
3664                 0x1a000, 0x1a06c,
3665                 0x1a0b0, 0x1a120,
3666                 0x1a128, 0x1a138,
3667                 0x1a190, 0x1a1c4,
3668                 0x1a1fc, 0x1a1fc,
3669                 0x1e040, 0x1e04c,
3670                 0x1e284, 0x1e28c,
3671                 0x1e2c0, 0x1e2c0,
3672                 0x1e2e0, 0x1e2e0,
3673                 0x1e300, 0x1e384,
3674                 0x1e3c0, 0x1e3c8,
3675                 0x1e440, 0x1e44c,
3676                 0x1e684, 0x1e68c,
3677                 0x1e6c0, 0x1e6c0,
3678                 0x1e6e0, 0x1e6e0,
3679                 0x1e700, 0x1e784,
3680                 0x1e7c0, 0x1e7c8,
3681                 0x1e840, 0x1e84c,
3682                 0x1ea84, 0x1ea8c,
3683                 0x1eac0, 0x1eac0,
3684                 0x1eae0, 0x1eae0,
3685                 0x1eb00, 0x1eb84,
3686                 0x1ebc0, 0x1ebc8,
3687                 0x1ec40, 0x1ec4c,
3688                 0x1ee84, 0x1ee8c,
3689                 0x1eec0, 0x1eec0,
3690                 0x1eee0, 0x1eee0,
3691                 0x1ef00, 0x1ef84,
3692                 0x1efc0, 0x1efc8,
3693                 0x1f040, 0x1f04c,
3694                 0x1f284, 0x1f28c,
3695                 0x1f2c0, 0x1f2c0,
3696                 0x1f2e0, 0x1f2e0,
3697                 0x1f300, 0x1f384,
3698                 0x1f3c0, 0x1f3c8,
3699                 0x1f440, 0x1f44c,
3700                 0x1f684, 0x1f68c,
3701                 0x1f6c0, 0x1f6c0,
3702                 0x1f6e0, 0x1f6e0,
3703                 0x1f700, 0x1f784,
3704                 0x1f7c0, 0x1f7c8,
3705                 0x1f840, 0x1f84c,
3706                 0x1fa84, 0x1fa8c,
3707                 0x1fac0, 0x1fac0,
3708                 0x1fae0, 0x1fae0,
3709                 0x1fb00, 0x1fb84,
3710                 0x1fbc0, 0x1fbc8,
3711                 0x1fc40, 0x1fc4c,
3712                 0x1fe84, 0x1fe8c,
3713                 0x1fec0, 0x1fec0,
3714                 0x1fee0, 0x1fee0,
3715                 0x1ff00, 0x1ff84,
3716                 0x1ffc0, 0x1ffc8,
3717                 0x20000, 0x2002c,
3718                 0x20100, 0x2013c,
3719                 0x20190, 0x201c8,
3720                 0x20200, 0x20318,
3721                 0x20400, 0x20528,
3722                 0x20540, 0x20614,
3723                 0x21000, 0x21040,
3724                 0x2104c, 0x21060,
3725                 0x210c0, 0x210ec,
3726                 0x21200, 0x21268,
3727                 0x21270, 0x21284,
3728                 0x212fc, 0x21388,
3729                 0x21400, 0x21404,
3730                 0x21500, 0x21518,
3731                 0x2152c, 0x2153c,
3732                 0x21550, 0x21554,
3733                 0x21600, 0x21600,
3734                 0x21608, 0x21628,
3735                 0x21630, 0x2163c,
3736                 0x21700, 0x2171c,
3737                 0x21780, 0x2178c,
3738                 0x21800, 0x21c38,
3739                 0x21c80, 0x21d7c,
3740                 0x21e00, 0x21e04,
3741                 0x22000, 0x2202c,
3742                 0x22100, 0x2213c,
3743                 0x22190, 0x221c8,
3744                 0x22200, 0x22318,
3745                 0x22400, 0x22528,
3746                 0x22540, 0x22614,
3747                 0x23000, 0x23040,
3748                 0x2304c, 0x23060,
3749                 0x230c0, 0x230ec,
3750                 0x23200, 0x23268,
3751                 0x23270, 0x23284,
3752                 0x232fc, 0x23388,
3753                 0x23400, 0x23404,
3754                 0x23500, 0x23518,
3755                 0x2352c, 0x2353c,
3756                 0x23550, 0x23554,
3757                 0x23600, 0x23600,
3758                 0x23608, 0x23628,
3759                 0x23630, 0x2363c,
3760                 0x23700, 0x2371c,
3761                 0x23780, 0x2378c,
3762                 0x23800, 0x23c38,
3763                 0x23c80, 0x23d7c,
3764                 0x23e00, 0x23e04,
3765                 0x24000, 0x2402c,
3766                 0x24100, 0x2413c,
3767                 0x24190, 0x241c8,
3768                 0x24200, 0x24318,
3769                 0x24400, 0x24528,
3770                 0x24540, 0x24614,
3771                 0x25000, 0x25040,
3772                 0x2504c, 0x25060,
3773                 0x250c0, 0x250ec,
3774                 0x25200, 0x25268,
3775                 0x25270, 0x25284,
3776                 0x252fc, 0x25388,
3777                 0x25400, 0x25404,
3778                 0x25500, 0x25518,
3779                 0x2552c, 0x2553c,
3780                 0x25550, 0x25554,
3781                 0x25600, 0x25600,
3782                 0x25608, 0x25628,
3783                 0x25630, 0x2563c,
3784                 0x25700, 0x2571c,
3785                 0x25780, 0x2578c,
3786                 0x25800, 0x25c38,
3787                 0x25c80, 0x25d7c,
3788                 0x25e00, 0x25e04,
3789                 0x26000, 0x2602c,
3790                 0x26100, 0x2613c,
3791                 0x26190, 0x261c8,
3792                 0x26200, 0x26318,
3793                 0x26400, 0x26528,
3794                 0x26540, 0x26614,
3795                 0x27000, 0x27040,
3796                 0x2704c, 0x27060,
3797                 0x270c0, 0x270ec,
3798                 0x27200, 0x27268,
3799                 0x27270, 0x27284,
3800                 0x272fc, 0x27388,
3801                 0x27400, 0x27404,
3802                 0x27500, 0x27518,
3803                 0x2752c, 0x2753c,
3804                 0x27550, 0x27554,
3805                 0x27600, 0x27600,
3806                 0x27608, 0x27628,
3807                 0x27630, 0x2763c,
3808                 0x27700, 0x2771c,
3809                 0x27780, 0x2778c,
3810                 0x27800, 0x27c38,
3811                 0x27c80, 0x27d7c,
3812                 0x27e00, 0x27e04
3813         };
3814         static const unsigned int t5_reg_ranges[] = {
3815                 0x1008, 0x1148,
3816                 0x1180, 0x11b4,
3817                 0x11fc, 0x123c,
3818                 0x1280, 0x173c,
3819                 0x1800, 0x18fc,
3820                 0x3000, 0x3028,
3821                 0x3060, 0x30d8,
3822                 0x30e0, 0x30fc,
3823                 0x3140, 0x357c,
3824                 0x35a8, 0x35cc,
3825                 0x35ec, 0x35ec,
3826                 0x3600, 0x5624,
3827                 0x56cc, 0x575c,
3828                 0x580c, 0x5814,
3829                 0x5890, 0x58bc,
3830                 0x5940, 0x59dc,
3831                 0x59fc, 0x5a18,
3832                 0x5a60, 0x5a9c,
3833                 0x5b94, 0x5bfc,
3834                 0x6000, 0x6040,
3835                 0x6058, 0x614c,
3836                 0x7700, 0x7798,
3837                 0x77c0, 0x78fc,
3838                 0x7b00, 0x7c54,
3839                 0x7d00, 0x7efc,
3840                 0x8dc0, 0x8de0,
3841                 0x8df8, 0x8e84,
3842                 0x8ea0, 0x8f84,
3843                 0x8fc0, 0x90f8,
3844                 0x9400, 0x9470,
3845                 0x9600, 0x96f4,
3846                 0x9800, 0x9808,
3847                 0x9820, 0x983c,
3848                 0x9850, 0x9864,
3849                 0x9c00, 0x9c6c,
3850                 0x9c80, 0x9cec,
3851                 0x9d00, 0x9d6c,
3852                 0x9d80, 0x9dec,
3853                 0x9e00, 0x9e6c,
3854                 0x9e80, 0x9eec,
3855                 0x9f00, 0x9f6c,
3856                 0x9f80, 0xa020,
3857                 0xd004, 0xd03c,
3858                 0xdfc0, 0xdfe0,
3859                 0xe000, 0x11088,
3860                 0x1109c, 0x11110,
3861                 0x11118, 0x1117c,
3862                 0x11190, 0x11204,
3863                 0x19040, 0x1906c,
3864                 0x19078, 0x19080,
3865                 0x1908c, 0x19124,
3866                 0x19150, 0x191b0,
3867                 0x191d0, 0x191e8,
3868                 0x19238, 0x19290,
3869                 0x193f8, 0x19474,
3870                 0x19490, 0x194cc,
3871                 0x194f0, 0x194f8,
3872                 0x19c00, 0x19c60,
3873                 0x19c94, 0x19e10,
3874                 0x19e50, 0x19f34,
3875                 0x19f40, 0x19f50,
3876                 0x19f90, 0x19fe4,
3877                 0x1a000, 0x1a06c,
3878                 0x1a0b0, 0x1a120,
3879                 0x1a128, 0x1a138,
3880                 0x1a190, 0x1a1c4,
3881                 0x1a1fc, 0x1a1fc,
3882                 0x1e008, 0x1e00c,
3883                 0x1e040, 0x1e04c,
3884                 0x1e284, 0x1e290,
3885                 0x1e2c0, 0x1e2c0,
3886                 0x1e2e0, 0x1e2e0,
3887                 0x1e300, 0x1e384,
3888                 0x1e3c0, 0x1e3c8,
3889                 0x1e408, 0x1e40c,
3890                 0x1e440, 0x1e44c,
3891                 0x1e684, 0x1e690,
3892                 0x1e6c0, 0x1e6c0,
3893                 0x1e6e0, 0x1e6e0,
3894                 0x1e700, 0x1e784,
3895                 0x1e7c0, 0x1e7c8,
3896                 0x1e808, 0x1e80c,
3897                 0x1e840, 0x1e84c,
3898                 0x1ea84, 0x1ea90,
3899                 0x1eac0, 0x1eac0,
3900                 0x1eae0, 0x1eae0,
3901                 0x1eb00, 0x1eb84,
3902                 0x1ebc0, 0x1ebc8,
3903                 0x1ec08, 0x1ec0c,
3904                 0x1ec40, 0x1ec4c,
3905                 0x1ee84, 0x1ee90,
3906                 0x1eec0, 0x1eec0,
3907                 0x1eee0, 0x1eee0,
3908                 0x1ef00, 0x1ef84,
3909                 0x1efc0, 0x1efc8,
3910                 0x1f008, 0x1f00c,
3911                 0x1f040, 0x1f04c,
3912                 0x1f284, 0x1f290,
3913                 0x1f2c0, 0x1f2c0,
3914                 0x1f2e0, 0x1f2e0,
3915                 0x1f300, 0x1f384,
3916                 0x1f3c0, 0x1f3c8,
3917                 0x1f408, 0x1f40c,
3918                 0x1f440, 0x1f44c,
3919                 0x1f684, 0x1f690,
3920                 0x1f6c0, 0x1f6c0,
3921                 0x1f6e0, 0x1f6e0,
3922                 0x1f700, 0x1f784,
3923                 0x1f7c0, 0x1f7c8,
3924                 0x1f808, 0x1f80c,
3925                 0x1f840, 0x1f84c,
3926                 0x1fa84, 0x1fa90,
3927                 0x1fac0, 0x1fac0,
3928                 0x1fae0, 0x1fae0,
3929                 0x1fb00, 0x1fb84,
3930                 0x1fbc0, 0x1fbc8,
3931                 0x1fc08, 0x1fc0c,
3932                 0x1fc40, 0x1fc4c,
3933                 0x1fe84, 0x1fe90,
3934                 0x1fec0, 0x1fec0,
3935                 0x1fee0, 0x1fee0,
3936                 0x1ff00, 0x1ff84,
3937                 0x1ffc0, 0x1ffc8,
3938                 0x30000, 0x30030,
3939                 0x30100, 0x30144,
3940                 0x30190, 0x301d0,
3941                 0x30200, 0x30318,
3942                 0x30400, 0x3052c,
3943                 0x30540, 0x3061c,
3944                 0x30800, 0x30834,
3945                 0x308c0, 0x30908,
3946                 0x30910, 0x309ac,
3947                 0x30a00, 0x30a2c,
3948                 0x30a44, 0x30a50,
3949                 0x30a74, 0x30c24,
3950                 0x30d00, 0x30d00,
3951                 0x30d08, 0x30d14,
3952                 0x30d1c, 0x30d20,
3953                 0x30d3c, 0x30d50,
3954                 0x31200, 0x3120c,
3955                 0x31220, 0x31220,
3956                 0x31240, 0x31240,
3957                 0x31600, 0x3160c,
3958                 0x31a00, 0x31a1c,
3959                 0x31e00, 0x31e20,
3960                 0x31e38, 0x31e3c,
3961                 0x31e80, 0x31e80,
3962                 0x31e88, 0x31ea8,
3963                 0x31eb0, 0x31eb4,
3964                 0x31ec8, 0x31ed4,
3965                 0x31fb8, 0x32004,
3966                 0x32200, 0x32200,
3967                 0x32208, 0x32240,
3968                 0x32248, 0x32280,
3969                 0x32288, 0x322c0,
3970                 0x322c8, 0x322fc,
3971                 0x32600, 0x32630,
3972                 0x32a00, 0x32abc,
3973                 0x32b00, 0x32b70,
3974                 0x33000, 0x33048,
3975                 0x33060, 0x3309c,
3976                 0x330f0, 0x33148,
3977                 0x33160, 0x3319c,
3978                 0x331f0, 0x332e4,
3979                 0x332f8, 0x333e4,
3980                 0x333f8, 0x33448,
3981                 0x33460, 0x3349c,
3982                 0x334f0, 0x33548,
3983                 0x33560, 0x3359c,
3984                 0x335f0, 0x336e4,
3985                 0x336f8, 0x337e4,
3986                 0x337f8, 0x337fc,
3987                 0x33814, 0x33814,
3988                 0x3382c, 0x3382c,
3989                 0x33880, 0x3388c,
3990                 0x338e8, 0x338ec,
3991                 0x33900, 0x33948,
3992                 0x33960, 0x3399c,
3993                 0x339f0, 0x33ae4,
3994                 0x33af8, 0x33b10,
3995                 0x33b28, 0x33b28,
3996                 0x33b3c, 0x33b50,
3997                 0x33bf0, 0x33c10,
3998                 0x33c28, 0x33c28,
3999                 0x33c3c, 0x33c50,
4000                 0x33cf0, 0x33cfc,
4001                 0x34000, 0x34030,
4002                 0x34100, 0x34144,
4003                 0x34190, 0x341d0,
4004                 0x34200, 0x34318,
4005                 0x34400, 0x3452c,
4006                 0x34540, 0x3461c,
4007                 0x34800, 0x34834,
4008                 0x348c0, 0x34908,
4009                 0x34910, 0x349ac,
4010                 0x34a00, 0x34a2c,
4011                 0x34a44, 0x34a50,
4012                 0x34a74, 0x34c24,
4013                 0x34d00, 0x34d00,
4014                 0x34d08, 0x34d14,
4015                 0x34d1c, 0x34d20,
4016                 0x34d3c, 0x34d50,
4017                 0x35200, 0x3520c,
4018                 0x35220, 0x35220,
4019                 0x35240, 0x35240,
4020                 0x35600, 0x3560c,
4021                 0x35a00, 0x35a1c,
4022                 0x35e00, 0x35e20,
4023                 0x35e38, 0x35e3c,
4024                 0x35e80, 0x35e80,
4025                 0x35e88, 0x35ea8,
4026                 0x35eb0, 0x35eb4,
4027                 0x35ec8, 0x35ed4,
4028                 0x35fb8, 0x36004,
4029                 0x36200, 0x36200,
4030                 0x36208, 0x36240,
4031                 0x36248, 0x36280,
4032                 0x36288, 0x362c0,
4033                 0x362c8, 0x362fc,
4034                 0x36600, 0x36630,
4035                 0x36a00, 0x36abc,
4036                 0x36b00, 0x36b70,
4037                 0x37000, 0x37048,
4038                 0x37060, 0x3709c,
4039                 0x370f0, 0x37148,
4040                 0x37160, 0x3719c,
4041                 0x371f0, 0x372e4,
4042                 0x372f8, 0x373e4,
4043                 0x373f8, 0x37448,
4044                 0x37460, 0x3749c,
4045                 0x374f0, 0x37548,
4046                 0x37560, 0x3759c,
4047                 0x375f0, 0x376e4,
4048                 0x376f8, 0x377e4,
4049                 0x377f8, 0x377fc,
4050                 0x37814, 0x37814,
4051                 0x3782c, 0x3782c,
4052                 0x37880, 0x3788c,
4053                 0x378e8, 0x378ec,
4054                 0x37900, 0x37948,
4055                 0x37960, 0x3799c,
4056                 0x379f0, 0x37ae4,
4057                 0x37af8, 0x37b10,
4058                 0x37b28, 0x37b28,
4059                 0x37b3c, 0x37b50,
4060                 0x37bf0, 0x37c10,
4061                 0x37c28, 0x37c28,
4062                 0x37c3c, 0x37c50,
4063                 0x37cf0, 0x37cfc,
4064                 0x38000, 0x38030,
4065                 0x38100, 0x38144,
4066                 0x38190, 0x381d0,
4067                 0x38200, 0x38318,
4068                 0x38400, 0x3852c,
4069                 0x38540, 0x3861c,
4070                 0x38800, 0x38834,
4071                 0x388c0, 0x38908,
4072                 0x38910, 0x389ac,
4073                 0x38a00, 0x38a2c,
4074                 0x38a44, 0x38a50,
4075                 0x38a74, 0x38c24,
4076                 0x38d00, 0x38d00,
4077                 0x38d08, 0x38d14,
4078                 0x38d1c, 0x38d20,
4079                 0x38d3c, 0x38d50,
4080                 0x39200, 0x3920c,
4081                 0x39220, 0x39220,
4082                 0x39240, 0x39240,
4083                 0x39600, 0x3960c,
4084                 0x39a00, 0x39a1c,
4085                 0x39e00, 0x39e20,
4086                 0x39e38, 0x39e3c,
4087                 0x39e80, 0x39e80,
4088                 0x39e88, 0x39ea8,
4089                 0x39eb0, 0x39eb4,
4090                 0x39ec8, 0x39ed4,
4091                 0x39fb8, 0x3a004,
4092                 0x3a200, 0x3a200,
4093                 0x3a208, 0x3a240,
4094                 0x3a248, 0x3a280,
4095                 0x3a288, 0x3a2c0,
4096                 0x3a2c8, 0x3a2fc,
4097                 0x3a600, 0x3a630,
4098                 0x3aa00, 0x3aabc,
4099                 0x3ab00, 0x3ab70,
4100                 0x3b000, 0x3b048,
4101                 0x3b060, 0x3b09c,
4102                 0x3b0f0, 0x3b148,
4103                 0x3b160, 0x3b19c,
4104                 0x3b1f0, 0x3b2e4,
4105                 0x3b2f8, 0x3b3e4,
4106                 0x3b3f8, 0x3b448,
4107                 0x3b460, 0x3b49c,
4108                 0x3b4f0, 0x3b548,
4109                 0x3b560, 0x3b59c,
4110                 0x3b5f0, 0x3b6e4,
4111                 0x3b6f8, 0x3b7e4,
4112                 0x3b7f8, 0x3b7fc,
4113                 0x3b814, 0x3b814,
4114                 0x3b82c, 0x3b82c,
4115                 0x3b880, 0x3b88c,
4116                 0x3b8e8, 0x3b8ec,
4117                 0x3b900, 0x3b948,
4118                 0x3b960, 0x3b99c,
4119                 0x3b9f0, 0x3bae4,
4120                 0x3baf8, 0x3bb10,
4121                 0x3bb28, 0x3bb28,
4122                 0x3bb3c, 0x3bb50,
4123                 0x3bbf0, 0x3bc10,
4124                 0x3bc28, 0x3bc28,
4125                 0x3bc3c, 0x3bc50,
4126                 0x3bcf0, 0x3bcfc,
4127                 0x3c000, 0x3c030,
4128                 0x3c100, 0x3c144,
4129                 0x3c190, 0x3c1d0,
4130                 0x3c200, 0x3c318,
4131                 0x3c400, 0x3c52c,
4132                 0x3c540, 0x3c61c,
4133                 0x3c800, 0x3c834,
4134                 0x3c8c0, 0x3c908,
4135                 0x3c910, 0x3c9ac,
4136                 0x3ca00, 0x3ca2c,
4137                 0x3ca44, 0x3ca50,
4138                 0x3ca74, 0x3cc24,
4139                 0x3cd00, 0x3cd00,
4140                 0x3cd08, 0x3cd14,
4141                 0x3cd1c, 0x3cd20,
4142                 0x3cd3c, 0x3cd50,
4143                 0x3d200, 0x3d20c,
4144                 0x3d220, 0x3d220,
4145                 0x3d240, 0x3d240,
4146                 0x3d600, 0x3d60c,
4147                 0x3da00, 0x3da1c,
4148                 0x3de00, 0x3de20,
4149                 0x3de38, 0x3de3c,
4150                 0x3de80, 0x3de80,
4151                 0x3de88, 0x3dea8,
4152                 0x3deb0, 0x3deb4,
4153                 0x3dec8, 0x3ded4,
4154                 0x3dfb8, 0x3e004,
4155                 0x3e200, 0x3e200,
4156                 0x3e208, 0x3e240,
4157                 0x3e248, 0x3e280,
4158                 0x3e288, 0x3e2c0,
4159                 0x3e2c8, 0x3e2fc,
4160                 0x3e600, 0x3e630,
4161                 0x3ea00, 0x3eabc,
4162                 0x3eb00, 0x3eb70,
4163                 0x3f000, 0x3f048,
4164                 0x3f060, 0x3f09c,
4165                 0x3f0f0, 0x3f148,
4166                 0x3f160, 0x3f19c,
4167                 0x3f1f0, 0x3f2e4,
4168                 0x3f2f8, 0x3f3e4,
4169                 0x3f3f8, 0x3f448,
4170                 0x3f460, 0x3f49c,
4171                 0x3f4f0, 0x3f548,
4172                 0x3f560, 0x3f59c,
4173                 0x3f5f0, 0x3f6e4,
4174                 0x3f6f8, 0x3f7e4,
4175                 0x3f7f8, 0x3f7fc,
4176                 0x3f814, 0x3f814,
4177                 0x3f82c, 0x3f82c,
4178                 0x3f880, 0x3f88c,
4179                 0x3f8e8, 0x3f8ec,
4180                 0x3f900, 0x3f948,
4181                 0x3f960, 0x3f99c,
4182                 0x3f9f0, 0x3fae4,
4183                 0x3faf8, 0x3fb10,
4184                 0x3fb28, 0x3fb28,
4185                 0x3fb3c, 0x3fb50,
4186                 0x3fbf0, 0x3fc10,
4187                 0x3fc28, 0x3fc28,
4188                 0x3fc3c, 0x3fc50,
4189                 0x3fcf0, 0x3fcfc,
4190                 0x40000, 0x4000c,
4191                 0x40040, 0x40068,
4192                 0x4007c, 0x40144,
4193                 0x40180, 0x4018c,
4194                 0x40200, 0x40298,
4195                 0x402ac, 0x4033c,
4196                 0x403f8, 0x403fc,
4197                 0x41304, 0x413c4,
4198                 0x41400, 0x4141c,
4199                 0x41480, 0x414d0,
4200                 0x44000, 0x44078,
4201                 0x440c0, 0x44278,
4202                 0x442c0, 0x44478,
4203                 0x444c0, 0x44678,
4204                 0x446c0, 0x44878,
4205                 0x448c0, 0x449fc,
4206                 0x45000, 0x45068,
4207                 0x45080, 0x45084,
4208                 0x450a0, 0x450b0,
4209                 0x45200, 0x45268,
4210                 0x45280, 0x45284,
4211                 0x452a0, 0x452b0,
4212                 0x460c0, 0x460e4,
4213                 0x47000, 0x4708c,
4214                 0x47200, 0x47250,
4215                 0x47400, 0x47420,
4216                 0x47600, 0x47618,
4217                 0x47800, 0x47814,
4218                 0x48000, 0x4800c,
4219                 0x48040, 0x48068,
4220                 0x4807c, 0x48144,
4221                 0x48180, 0x4818c,
4222                 0x48200, 0x48298,
4223                 0x482ac, 0x4833c,
4224                 0x483f8, 0x483fc,
4225                 0x49304, 0x493c4,
4226                 0x49400, 0x4941c,
4227                 0x49480, 0x494d0,
4228                 0x4c000, 0x4c078,
4229                 0x4c0c0, 0x4c278,
4230                 0x4c2c0, 0x4c478,
4231                 0x4c4c0, 0x4c678,
4232                 0x4c6c0, 0x4c878,
4233                 0x4c8c0, 0x4c9fc,
4234                 0x4d000, 0x4d068,
4235                 0x4d080, 0x4d084,
4236                 0x4d0a0, 0x4d0b0,
4237                 0x4d200, 0x4d268,
4238                 0x4d280, 0x4d284,
4239                 0x4d2a0, 0x4d2b0,
4240                 0x4e0c0, 0x4e0e4,
4241                 0x4f000, 0x4f08c,
4242                 0x4f200, 0x4f250,
4243                 0x4f400, 0x4f420,
4244                 0x4f600, 0x4f618,
4245                 0x4f800, 0x4f814,
4246                 0x50000, 0x500cc,
4247                 0x50400, 0x50400,
4248                 0x50800, 0x508cc,
4249                 0x50c00, 0x50c00,
4250                 0x51000, 0x5101c,
4251                 0x51300, 0x51308,
4252         };
4253
4254         if (is_t4(sc)) {
4255                 reg_ranges = &t4_reg_ranges[0];
4256                 n = nitems(t4_reg_ranges);
4257         } else {
4258                 reg_ranges = &t5_reg_ranges[0];
4259                 n = nitems(t5_reg_ranges);
4260         }
4261
4262         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4263         for (i = 0; i < n; i += 2)
4264                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4265 }
4266
4267 static void
4268 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4269 {
4270         struct ifnet *ifp = pi->ifp;
4271         struct sge_txq *txq;
4272         int i, drops;
4273         struct port_stats *s = &pi->stats;
4274         struct timeval tv;
4275         const struct timeval interval = {0, 250000};    /* 250ms */
4276
4277         getmicrotime(&tv);
4278         timevalsub(&tv, &interval);
4279         if (timevalcmp(&tv, &pi->last_refreshed, <))
4280                 return;
4281
4282         t4_get_port_stats(sc, pi->tx_chan, s);
4283
4284         ifp->if_opackets = s->tx_frames - s->tx_pause;
4285         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4286         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4287         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4288         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4289         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4290         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4291             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4292             s->rx_trunc3;
4293         for (i = 0; i < NCHAN; i++) {
4294                 if (pi->rx_chan_map & (1 << i)) {
4295                         uint32_t v;
4296
4297                         mtx_lock(&sc->regwin_lock);
4298                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4299                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4300                         mtx_unlock(&sc->regwin_lock);
4301                         ifp->if_iqdrops += v;
4302                 }
4303         }
4304
4305         drops = s->tx_drop;
4306         for_each_txq(pi, i, txq)
4307                 drops += counter_u64_fetch(txq->r->drops);
4308         ifp->if_snd.ifq_drops = drops;
4309
4310         ifp->if_oerrors = s->tx_error_frames;
4311         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4312             s->rx_fcs_err + s->rx_len_err;
4313
4314         getmicrotime(&pi->last_refreshed);
4315 }
4316
4317 static void
4318 cxgbe_tick(void *arg)
4319 {
4320         struct port_info *pi = arg;
4321         struct adapter *sc = pi->adapter;
4322         struct ifnet *ifp = pi->ifp;
4323
4324         PORT_LOCK(pi);
4325         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4326                 PORT_UNLOCK(pi);
4327                 return; /* without scheduling another callout */
4328         }
4329
4330         cxgbe_refresh_stats(sc, pi);
4331
4332         callout_schedule(&pi->tick, hz);
4333         PORT_UNLOCK(pi);
4334 }
4335
4336 static void
4337 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4338 {
4339         struct ifnet *vlan;
4340
4341         if (arg != ifp || ifp->if_type != IFT_ETHER)
4342                 return;
4343
4344         vlan = VLAN_DEVAT(ifp, vid);
4345         VLAN_SETCOOKIE(vlan, ifp);
4346 }
4347
4348 static int
4349 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4350 {
4351
4352 #ifdef INVARIANTS
4353         panic("%s: opcode 0x%02x on iq %p with payload %p",
4354             __func__, rss->opcode, iq, m);
4355 #else
4356         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4357             __func__, rss->opcode, iq, m);
4358         m_freem(m);
4359 #endif
4360         return (EDOOFUS);
4361 }
4362
4363 int
4364 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4365 {
4366         uintptr_t *loc, new;
4367
4368         if (opcode >= nitems(sc->cpl_handler))
4369                 return (EINVAL);
4370
4371         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4372         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4373         atomic_store_rel_ptr(loc, new);
4374
4375         return (0);
4376 }
4377
4378 static int
4379 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4380 {
4381
4382 #ifdef INVARIANTS
4383         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4384 #else
4385         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4386             __func__, iq, ctrl);
4387 #endif
4388         return (EDOOFUS);
4389 }
4390
4391 int
4392 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4393 {
4394         uintptr_t *loc, new;
4395
4396         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4397         loc = (uintptr_t *) &sc->an_handler;
4398         atomic_store_rel_ptr(loc, new);
4399
4400         return (0);
4401 }
4402
4403 static int
4404 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4405 {
4406         const struct cpl_fw6_msg *cpl =
4407             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4408
4409 #ifdef INVARIANTS
4410         panic("%s: fw_msg type %d", __func__, cpl->type);
4411 #else
4412         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4413 #endif
4414         return (EDOOFUS);
4415 }
4416
4417 int
4418 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4419 {
4420         uintptr_t *loc, new;
4421
4422         if (type >= nitems(sc->fw_msg_handler))
4423                 return (EINVAL);
4424
4425         /*
4426          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4427          * handler dispatch table.  Reject any attempt to install a handler for
4428          * this subtype.
4429          */
4430         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4431                 return (EINVAL);
4432
4433         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4434         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4435         atomic_store_rel_ptr(loc, new);
4436
4437         return (0);
4438 }
4439
4440 static int
4441 t4_sysctls(struct adapter *sc)
4442 {
4443         struct sysctl_ctx_list *ctx;
4444         struct sysctl_oid *oid;
4445         struct sysctl_oid_list *children, *c0;
4446         static char *caps[] = {
4447                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4448                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4449                     "\6HASHFILTER\7ETHOFLD",
4450                 "\20\1TOE",                             /* caps[2] toecaps */
4451                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4452                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4453                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4454                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4455                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4456                     "\4PO_INITIAOR\5PO_TARGET"
4457         };
4458         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4459
4460         ctx = device_get_sysctl_ctx(sc->dev);
4461
4462         /*
4463          * dev.t4nex.X.
4464          */
4465         oid = device_get_sysctl_tree(sc->dev);
4466         c0 = children = SYSCTL_CHILDREN(oid);
4467
4468         sc->sc_do_rxcopy = 1;
4469         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4470             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4471
4472         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4473             sc->params.nports, "# of ports");
4474
4475         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4476             NULL, chip_rev(sc), "chip hardware revision");
4477
4478         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4479             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4480
4481         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4482             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4483
4484         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4485             sc->cfcsum, "config file checksum");
4486
4487         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4488             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4489             sysctl_bitfield, "A", "available doorbells");
4490
4491         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4492             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4493             sysctl_bitfield, "A", "available link capabilities");
4494
4495         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4496             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4497             sysctl_bitfield, "A", "available NIC capabilities");
4498
4499         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4500             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4501             sysctl_bitfield, "A", "available TCP offload capabilities");
4502
4503         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4504             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4505             sysctl_bitfield, "A", "available RDMA capabilities");
4506
4507         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4508             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4509             sysctl_bitfield, "A", "available iSCSI capabilities");
4510
4511         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4512             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4513             sysctl_bitfield, "A", "available FCoE capabilities");
4514
4515         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4516             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4517
4518         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4519             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4520             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4521             "interrupt holdoff timer values (us)");
4522
4523         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4524             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4525             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4526             "interrupt holdoff packet counter values");
4527
4528         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4529             NULL, sc->tids.nftids, "number of filters");
4530
4531         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4532             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4533             "chip temperature (in Celsius)");
4534
4535         t4_sge_sysctls(sc, ctx, children);
4536
4537         sc->lro_timeout = 100;
4538         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4539             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4540
4541 #ifdef SBUF_DRAIN
4542         /*
4543          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4544          */
4545         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4546             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4547             "logs and miscellaneous information");
4548         children = SYSCTL_CHILDREN(oid);
4549
4550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4551             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4552             sysctl_cctrl, "A", "congestion control");
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4555             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4556             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4559             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4560             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4563             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4564             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4565
4566         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4567             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4568             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4569
4570         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4571             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4572             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4575             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4576             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580             sysctl_cim_la, "A", "CIM logic analyzer");
4581
4582         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4583             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4588             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4592             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4596             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4600             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4604             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4608             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4609
4610         if (is_t5(sc)) {
4611                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4612                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4613                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4614
4615                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4616                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4617                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4618         }
4619
4620         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4621             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4622             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4623
4624         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4625             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4626             sysctl_cim_qcfg, "A", "CIM queue configuration");
4627
4628         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4629             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4630             sysctl_cpl_stats, "A", "CPL statistics");
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638             sysctl_devlog, "A", "firmware's device log");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4642             sysctl_fcoe_stats, "A", "FCoE statistics");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646             sysctl_hw_sched, "A", "hardware scheduler ");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650             sysctl_l2t, "A", "hardware L2 table");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4654             sysctl_lb_stats, "A", "loopback statistics");
4655
4656         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4657             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4658             sysctl_meminfo, "A", "memory regions");
4659
4660         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4661             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4662             sysctl_mps_tcam, "A", "MPS TCAM entries");
4663
4664         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4665             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4666             sysctl_path_mtus, "A", "path MTUs");
4667
4668         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4669             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4670             sysctl_pm_stats, "A", "PM statistics");
4671
4672         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4673             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4674             sysctl_rdma_stats, "A", "RDMA statistics");
4675
4676         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4677             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4678             sysctl_tcp_stats, "A", "TCP statistics");
4679
4680         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4681             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4682             sysctl_tids, "A", "TID information");
4683
4684         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4685             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686             sysctl_tp_err_stats, "A", "TP error statistics");
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_tp_la, "A", "TP logic analyzer");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_tx_rate, "A", "Tx rate");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4699
4700         if (is_t5(sc)) {
4701                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4702                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703                     sysctl_wcwr_stats, "A", "write combined work requests");
4704         }
4705 #endif
4706
4707 #ifdef TCP_OFFLOAD
4708         if (is_offload(sc)) {
4709                 /*
4710                  * dev.t4nex.X.toe.
4711                  */
4712                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4713                     NULL, "TOE parameters");
4714                 children = SYSCTL_CHILDREN(oid);
4715
4716                 sc->tt.sndbuf = 256 * 1024;
4717                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4718                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4719
4720                 sc->tt.ddp = 0;
4721                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4722                     &sc->tt.ddp, 0, "DDP allowed");
4723
4724                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4725                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4726                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4727
4728                 sc->tt.ddp_thres =
4729                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4730                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4731                     &sc->tt.ddp_thres, 0, "DDP threshold");
4732
4733                 sc->tt.rx_coalesce = 1;
4734                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4735                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4736
4737                 sc->tt.tx_align = 1;
4738                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4739                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4740         }
4741 #endif
4742
4743
4744         return (0);
4745 }
4746
4747 static int
4748 cxgbe_sysctls(struct port_info *pi)
4749 {
4750         struct sysctl_ctx_list *ctx;
4751         struct sysctl_oid *oid;
4752         struct sysctl_oid_list *children;
4753         struct adapter *sc = pi->adapter;
4754
4755         ctx = device_get_sysctl_ctx(pi->dev);
4756
4757         /*
4758          * dev.cxgbe.X.
4759          */
4760         oid = device_get_sysctl_tree(pi->dev);
4761         children = SYSCTL_CHILDREN(oid);
4762
4763         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4764            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4765         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4766                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4767                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4768                     "PHY temperature (in Celsius)");
4769                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4770                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4771                     "PHY firmware version");
4772         }
4773         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4774             &pi->nrxq, 0, "# of rx queues");
4775         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4776             &pi->ntxq, 0, "# of tx queues");
4777         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4778             &pi->first_rxq, 0, "index of first rx queue");
4779         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4780             &pi->first_txq, 0, "index of first tx queue");
4781         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4782             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4783             "Reserve queue 0 for non-flowid packets");
4784
4785 #ifdef TCP_OFFLOAD
4786         if (is_offload(sc)) {
4787                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4788                     &pi->nofldrxq, 0,
4789                     "# of rx queues for offloaded TCP connections");
4790                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4791                     &pi->nofldtxq, 0,
4792                     "# of tx queues for offloaded TCP connections");
4793                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4794                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4795                     "index of first TOE rx queue");
4796                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4797                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4798                     "index of first TOE tx queue");
4799         }
4800 #endif
4801 #ifdef DEV_NETMAP
4802         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4803             &pi->nnmrxq, 0, "# of rx queues for netmap");
4804         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4805             &pi->nnmtxq, 0, "# of tx queues for netmap");
4806         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4807             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4808             "index of first netmap rx queue");
4809         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4810             CTLFLAG_RD, &pi->first_nm_txq, 0,
4811             "index of first netmap tx queue");
4812 #endif
4813
4814         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4815             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4816             "holdoff timer index");
4817         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4818             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4819             "holdoff packet counter index");
4820
4821         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4822             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4823             "rx queue size");
4824         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4825             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4826             "tx queue size");
4827
4828         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4829             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4830             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4831
4832         /*
4833          * dev.cxgbe.X.stats.
4834          */
4835         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4836             NULL, "port statistics");
4837         children = SYSCTL_CHILDREN(oid);
4838         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4839             &pi->tx_parse_error, 0,
4840             "# of tx packets with invalid length or # of segments");
4841
4842 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4843         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4844             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4845             sysctl_handle_t4_reg64, "QU", desc)
4846
4847         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4848             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4849         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4850             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4851         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4853         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4855         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4857         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4858             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4859         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4860             "# of tx frames in this range",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4862         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4863             "# of tx frames in this range",
4864             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4865         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4866             "# of tx frames in this range",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4868         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4869             "# of tx frames in this range",
4870             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4871         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4872             "# of tx frames in this range",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4874         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4875             "# of tx frames in this range",
4876             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4877         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4878             "# of tx frames in this range",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4880         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4882         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4884         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4886         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4887             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4888         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4890         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4891             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4892         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4893             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4894         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4895             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4896         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4897             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4898         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4899             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4900
4901         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4913         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4915         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4916             "# of frames received with bad FCS",
4917             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4918         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4919             "# of frames received with length error",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4921         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4923         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4925         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4926             "# of rx frames in this range",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4928         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4929             "# of rx frames in this range",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4931         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4932             "# of rx frames in this range",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4934         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4935             "# of rx frames in this range",
4936             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4937         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4938             "# of rx frames in this range",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4940         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4941             "# of rx frames in this range",
4942             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4943         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4944             "# of rx frames in this range",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4946         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4948         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4950         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4952         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4954         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4956         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4958         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4960         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4962         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4964
4965 #undef SYSCTL_ADD_T4_REG64
4966
4967 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4968         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4969             &pi->stats.name, desc)
4970
4971         /* We get these from port_stats and they may be stale by upto 1s */
4972         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4973             "# drops due to buffer-group 0 overflows");
4974         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4975             "# drops due to buffer-group 1 overflows");
4976         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4977             "# drops due to buffer-group 2 overflows");
4978         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4979             "# drops due to buffer-group 3 overflows");
4980         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4981             "# of buffer-group 0 truncated packets");
4982         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4983             "# of buffer-group 1 truncated packets");
4984         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4985             "# of buffer-group 2 truncated packets");
4986         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4987             "# of buffer-group 3 truncated packets");
4988
4989 #undef SYSCTL_ADD_T4_PORTSTAT
4990
4991         return (0);
4992 }
4993
4994 static int
4995 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4996 {
4997         int rc, *i, space = 0;
4998         struct sbuf sb;
4999
5000         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5001         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5002                 if (space)
5003                         sbuf_printf(&sb, " ");
5004                 sbuf_printf(&sb, "%d", *i);
5005                 space = 1;
5006         }
5007         sbuf_finish(&sb);
5008         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5009         sbuf_delete(&sb);
5010         return (rc);
5011 }
5012
5013 static int
5014 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5015 {
5016         int rc;
5017         struct sbuf *sb;
5018
5019         rc = sysctl_wire_old_buffer(req, 0);
5020         if (rc != 0)
5021                 return(rc);
5022
5023         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5024         if (sb == NULL)
5025                 return (ENOMEM);
5026
5027         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5028         rc = sbuf_finish(sb);
5029         sbuf_delete(sb);
5030
5031         return (rc);
5032 }
5033
5034 static int
5035 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5036 {
5037         struct port_info *pi = arg1;
5038         int op = arg2;
5039         struct adapter *sc = pi->adapter;
5040         u_int v;
5041         int rc;
5042
5043         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5044         if (rc)
5045                 return (rc);
5046         /* XXX: magic numbers */
5047         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5048             &v);
5049         end_synchronized_op(sc, 0);
5050         if (rc)
5051                 return (rc);
5052         if (op == 0)
5053                 v /= 256;
5054
5055         rc = sysctl_handle_int(oidp, &v, 0, req);
5056         return (rc);
5057 }
5058
5059 static int
5060 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5061 {
5062         struct port_info *pi = arg1;
5063         int rc, val;
5064
5065         val = pi->rsrv_noflowq;
5066         rc = sysctl_handle_int(oidp, &val, 0, req);
5067         if (rc != 0 || req->newptr == NULL)
5068                 return (rc);
5069
5070         if ((val >= 1) && (pi->ntxq > 1))
5071                 pi->rsrv_noflowq = 1;
5072         else
5073                 pi->rsrv_noflowq = 0;
5074
5075         return (rc);
5076 }
5077
5078 static int
5079 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5080 {
5081         struct port_info *pi = arg1;
5082         struct adapter *sc = pi->adapter;
5083         int idx, rc, i;
5084         struct sge_rxq *rxq;
5085 #ifdef TCP_OFFLOAD
5086         struct sge_ofld_rxq *ofld_rxq;
5087 #endif
5088         uint8_t v;
5089
5090         idx = pi->tmr_idx;
5091
5092         rc = sysctl_handle_int(oidp, &idx, 0, req);
5093         if (rc != 0 || req->newptr == NULL)
5094                 return (rc);
5095
5096         if (idx < 0 || idx >= SGE_NTIMERS)
5097                 return (EINVAL);
5098
5099         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5100             "t4tmr");
5101         if (rc)
5102                 return (rc);
5103
5104         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5105         for_each_rxq(pi, i, rxq) {
5106 #ifdef atomic_store_rel_8
5107                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5108 #else
5109                 rxq->iq.intr_params = v;
5110 #endif
5111         }
5112 #ifdef TCP_OFFLOAD
5113         for_each_ofld_rxq(pi, i, ofld_rxq) {
5114 #ifdef atomic_store_rel_8
5115                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5116 #else
5117                 ofld_rxq->iq.intr_params = v;
5118 #endif
5119         }
5120 #endif
5121         pi->tmr_idx = idx;
5122
5123         end_synchronized_op(sc, LOCK_HELD);
5124         return (0);
5125 }
5126
5127 static int
5128 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5129 {
5130         struct port_info *pi = arg1;
5131         struct adapter *sc = pi->adapter;
5132         int idx, rc;
5133
5134         idx = pi->pktc_idx;
5135
5136         rc = sysctl_handle_int(oidp, &idx, 0, req);
5137         if (rc != 0 || req->newptr == NULL)
5138                 return (rc);
5139
5140         if (idx < -1 || idx >= SGE_NCOUNTERS)
5141                 return (EINVAL);
5142
5143         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5144             "t4pktc");
5145         if (rc)
5146                 return (rc);
5147
5148         if (pi->flags & PORT_INIT_DONE)
5149                 rc = EBUSY; /* cannot be changed once the queues are created */
5150         else
5151                 pi->pktc_idx = idx;
5152
5153         end_synchronized_op(sc, LOCK_HELD);
5154         return (rc);
5155 }
5156
5157 static int
5158 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5159 {
5160         struct port_info *pi = arg1;
5161         struct adapter *sc = pi->adapter;
5162         int qsize, rc;
5163
5164         qsize = pi->qsize_rxq;
5165
5166         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5167         if (rc != 0 || req->newptr == NULL)
5168                 return (rc);
5169
5170         if (qsize < 128 || (qsize & 7))
5171                 return (EINVAL);
5172
5173         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5174             "t4rxqs");
5175         if (rc)
5176                 return (rc);
5177
5178         if (pi->flags & PORT_INIT_DONE)
5179                 rc = EBUSY; /* cannot be changed once the queues are created */
5180         else
5181                 pi->qsize_rxq = qsize;
5182
5183         end_synchronized_op(sc, LOCK_HELD);
5184         return (rc);
5185 }
5186
5187 static int
5188 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5189 {
5190         struct port_info *pi = arg1;
5191         struct adapter *sc = pi->adapter;
5192         int qsize, rc;
5193
5194         qsize = pi->qsize_txq;
5195
5196         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5197         if (rc != 0 || req->newptr == NULL)
5198                 return (rc);
5199
5200         if (qsize < 128 || qsize > 65536)
5201                 return (EINVAL);
5202
5203         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5204             "t4txqs");
5205         if (rc)
5206                 return (rc);
5207
5208         if (pi->flags & PORT_INIT_DONE)
5209                 rc = EBUSY; /* cannot be changed once the queues are created */
5210         else
5211                 pi->qsize_txq = qsize;
5212
5213         end_synchronized_op(sc, LOCK_HELD);
5214         return (rc);
5215 }
5216
5217 static int
5218 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5219 {
5220         struct port_info *pi = arg1;
5221         struct adapter *sc = pi->adapter;
5222         struct link_config *lc = &pi->link_cfg;
5223         int rc;
5224
5225         if (req->newptr == NULL) {
5226                 struct sbuf *sb;
5227                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5228
5229                 rc = sysctl_wire_old_buffer(req, 0);
5230                 if (rc != 0)
5231                         return(rc);
5232
5233                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5234                 if (sb == NULL)
5235                         return (ENOMEM);
5236
5237                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5238                 rc = sbuf_finish(sb);
5239                 sbuf_delete(sb);
5240         } else {
5241                 char s[2];
5242                 int n;
5243
5244                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5245                 s[1] = 0;
5246
5247                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5248                 if (rc != 0)
5249                         return(rc);
5250
5251                 if (s[1] != 0)
5252                         return (EINVAL);
5253                 if (s[0] < '0' || s[0] > '9')
5254                         return (EINVAL);        /* not a number */
5255                 n = s[0] - '0';
5256                 if (n & ~(PAUSE_TX | PAUSE_RX))
5257                         return (EINVAL);        /* some other bit is set too */
5258
5259                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5260                 if (rc)
5261                         return (rc);
5262                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5263                         int link_ok = lc->link_ok;
5264
5265                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5266                         lc->requested_fc |= n;
5267                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5268                         lc->link_ok = link_ok;  /* restore */
5269                 }
5270                 end_synchronized_op(sc, 0);
5271         }
5272
5273         return (rc);
5274 }
5275
5276 static int
5277 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5278 {
5279         struct adapter *sc = arg1;
5280         int reg = arg2;
5281         uint64_t val;
5282
5283         val = t4_read_reg64(sc, reg);
5284
5285         return (sysctl_handle_64(oidp, &val, 0, req));
5286 }
5287
5288 static int
5289 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5290 {
5291         struct adapter *sc = arg1;
5292         int rc, t;
5293         uint32_t param, val;
5294
5295         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5296         if (rc)
5297                 return (rc);
5298         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5299             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5300             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5301         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5302         end_synchronized_op(sc, 0);
5303         if (rc)
5304                 return (rc);
5305
5306         /* unknown is returned as 0 but we display -1 in that case */
5307         t = val == 0 ? -1 : val;
5308
5309         rc = sysctl_handle_int(oidp, &t, 0, req);
5310         return (rc);
5311 }
5312
5313 #ifdef SBUF_DRAIN
5314 static int
5315 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5316 {
5317         struct adapter *sc = arg1;
5318         struct sbuf *sb;
5319         int rc, i;
5320         uint16_t incr[NMTUS][NCCTRL_WIN];
5321         static const char *dec_fac[] = {
5322                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5323                 "0.9375"
5324         };
5325
5326         rc = sysctl_wire_old_buffer(req, 0);
5327         if (rc != 0)
5328                 return (rc);
5329
5330         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5331         if (sb == NULL)
5332                 return (ENOMEM);
5333
5334         t4_read_cong_tbl(sc, incr);
5335
5336         for (i = 0; i < NCCTRL_WIN; ++i) {
5337                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5338                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5339                     incr[5][i], incr[6][i], incr[7][i]);
5340                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5341                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5342                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5343                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5344         }
5345
5346         rc = sbuf_finish(sb);
5347         sbuf_delete(sb);
5348
5349         return (rc);
5350 }
5351
5352 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5353         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5354         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5355         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5356 };
5357
5358 static int
5359 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5360 {
5361         struct adapter *sc = arg1;
5362         struct sbuf *sb;
5363         int rc, i, n, qid = arg2;
5364         uint32_t *buf, *p;
5365         char *qtype;
5366         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5367
5368         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5369             ("%s: bad qid %d\n", __func__, qid));
5370
5371         if (qid < CIM_NUM_IBQ) {
5372                 /* inbound queue */
5373                 qtype = "IBQ";
5374                 n = 4 * CIM_IBQ_SIZE;
5375                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5376                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5377         } else {
5378                 /* outbound queue */
5379                 qtype = "OBQ";
5380                 qid -= CIM_NUM_IBQ;
5381                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5382                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5383                 rc = t4_read_cim_obq(sc, qid, buf, n);
5384         }
5385
5386         if (rc < 0) {
5387                 rc = -rc;
5388                 goto done;
5389         }
5390         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5391
5392         rc = sysctl_wire_old_buffer(req, 0);
5393         if (rc != 0)
5394                 goto done;
5395
5396         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5397         if (sb == NULL) {
5398                 rc = ENOMEM;
5399                 goto done;
5400         }
5401
5402         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5403         for (i = 0, p = buf; i < n; i += 16, p += 4)
5404                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5405                     p[2], p[3]);
5406
5407         rc = sbuf_finish(sb);
5408         sbuf_delete(sb);
5409 done:
5410         free(buf, M_CXGBE);
5411         return (rc);
5412 }
5413
5414 static int
5415 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5416 {
5417         struct adapter *sc = arg1;
5418         u_int cfg;
5419         struct sbuf *sb;
5420         uint32_t *buf, *p;
5421         int rc;
5422
5423         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5424         if (rc != 0)
5425                 return (rc);
5426
5427         rc = sysctl_wire_old_buffer(req, 0);
5428         if (rc != 0)
5429                 return (rc);
5430
5431         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5432         if (sb == NULL)
5433                 return (ENOMEM);
5434
5435         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5436             M_ZERO | M_WAITOK);
5437
5438         rc = -t4_cim_read_la(sc, buf, NULL);
5439         if (rc != 0)
5440                 goto done;
5441
5442         sbuf_printf(sb, "Status   Data      PC%s",
5443             cfg & F_UPDBGLACAPTPCONLY ? "" :
5444             "     LS0Stat  LS0Addr             LS0Data");
5445
5446         KASSERT((sc->params.cim_la_size & 7) == 0,
5447             ("%s: p will walk off the end of buf", __func__));
5448
5449         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5450                 if (cfg & F_UPDBGLACAPTPCONLY) {
5451                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5452                             p[6], p[7]);
5453                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5454                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5455                             p[4] & 0xff, p[5] >> 8);
5456                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5457                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5458                             p[1] & 0xf, p[2] >> 4);
5459                 } else {
5460                         sbuf_printf(sb,
5461                             "\n  %02x   %x%07x %x%07x %08x %08x "
5462                             "%08x%08x%08x%08x",
5463                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5464                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5465                             p[6], p[7]);
5466                 }
5467         }
5468
5469         rc = sbuf_finish(sb);
5470         sbuf_delete(sb);
5471 done:
5472         free(buf, M_CXGBE);
5473         return (rc);
5474 }
5475
5476 static int
5477 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5478 {
5479         struct adapter *sc = arg1;
5480         u_int i;
5481         struct sbuf *sb;
5482         uint32_t *buf, *p;
5483         int rc;
5484
5485         rc = sysctl_wire_old_buffer(req, 0);
5486         if (rc != 0)
5487                 return (rc);
5488
5489         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5490         if (sb == NULL)
5491                 return (ENOMEM);
5492
5493         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5494             M_ZERO | M_WAITOK);
5495
5496         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5497         p = buf;
5498
5499         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5500                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5501                     p[1], p[0]);
5502         }
5503
5504         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5505         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5506                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5507                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5508                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5509                     (p[1] >> 2) | ((p[2] & 3) << 30),
5510                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5511                     p[0] & 1);
5512         }
5513
5514         rc = sbuf_finish(sb);
5515         sbuf_delete(sb);
5516         free(buf, M_CXGBE);
5517         return (rc);
5518 }
5519
5520 static int
5521 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5522 {
5523         struct adapter *sc = arg1;
5524         u_int i;
5525         struct sbuf *sb;
5526         uint32_t *buf, *p;
5527         int rc;
5528
5529         rc = sysctl_wire_old_buffer(req, 0);
5530         if (rc != 0)
5531                 return (rc);
5532
5533         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5534         if (sb == NULL)
5535                 return (ENOMEM);
5536
5537         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5538             M_ZERO | M_WAITOK);
5539
5540         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5541         p = buf;
5542
5543         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5544         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5545                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5546                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5547                     p[4], p[3], p[2], p[1], p[0]);
5548         }
5549
5550         sbuf_printf(sb, "\n\nCntl ID               Data");
5551         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5552                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5553                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5554         }
5555
5556         rc = sbuf_finish(sb);
5557         sbuf_delete(sb);
5558         free(buf, M_CXGBE);
5559         return (rc);
5560 }
5561
5562 static int
5563 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5564 {
5565         struct adapter *sc = arg1;
5566         struct sbuf *sb;
5567         int rc, i;
5568         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5569         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5570         uint16_t thres[CIM_NUM_IBQ];
5571         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5572         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5573         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5574
5575         if (is_t4(sc)) {
5576                 cim_num_obq = CIM_NUM_OBQ;
5577                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5578                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5579         } else {
5580                 cim_num_obq = CIM_NUM_OBQ_T5;
5581                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5582                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5583         }
5584         nq = CIM_NUM_IBQ + cim_num_obq;
5585
5586         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5587         if (rc == 0)
5588                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5589         if (rc != 0)
5590                 return (rc);
5591
5592         t4_read_cimq_cfg(sc, base, size, thres);
5593
5594         rc = sysctl_wire_old_buffer(req, 0);
5595         if (rc != 0)
5596                 return (rc);
5597
5598         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5599         if (sb == NULL)
5600                 return (ENOMEM);
5601
5602         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5603
5604         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5605                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5606                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5607                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5608                     G_QUEREMFLITS(p[2]) * 16);
5609         for ( ; i < nq; i++, p += 4, wr += 2)
5610                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5611                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5612                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5613                     G_QUEREMFLITS(p[2]) * 16);
5614
5615         rc = sbuf_finish(sb);
5616         sbuf_delete(sb);
5617
5618         return (rc);
5619 }
5620
5621 static int
5622 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5623 {
5624         struct adapter *sc = arg1;
5625         struct sbuf *sb;
5626         int rc;
5627         struct tp_cpl_stats stats;
5628
5629         rc = sysctl_wire_old_buffer(req, 0);
5630         if (rc != 0)
5631                 return (rc);
5632
5633         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5634         if (sb == NULL)
5635                 return (ENOMEM);
5636
5637         t4_tp_get_cpl_stats(sc, &stats);
5638
5639         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5640             "channel 3\n");
5641         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5642                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5643         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5644                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5645
5646         rc = sbuf_finish(sb);
5647         sbuf_delete(sb);
5648
5649         return (rc);
5650 }
5651
5652 static int
5653 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5654 {
5655         struct adapter *sc = arg1;
5656         struct sbuf *sb;
5657         int rc;
5658         struct tp_usm_stats stats;
5659
5660         rc = sysctl_wire_old_buffer(req, 0);
5661         if (rc != 0)
5662                 return(rc);
5663
5664         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5665         if (sb == NULL)
5666                 return (ENOMEM);
5667
5668         t4_get_usm_stats(sc, &stats);
5669
5670         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5671         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5672         sbuf_printf(sb, "Drops:  %u", stats.drops);
5673
5674         rc = sbuf_finish(sb);
5675         sbuf_delete(sb);
5676
5677         return (rc);
5678 }
5679
5680 const char *devlog_level_strings[] = {
5681         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5682         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5683         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5684         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5685         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5686         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5687 };
5688
5689 const char *devlog_facility_strings[] = {
5690         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5691         [FW_DEVLOG_FACILITY_CF]         = "CF",
5692         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5693         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5694         [FW_DEVLOG_FACILITY_RES]        = "RES",
5695         [FW_DEVLOG_FACILITY_HW]         = "HW",
5696         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5697         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5698         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5699         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5700         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5701         [FW_DEVLOG_FACILITY_VI]         = "VI",
5702         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5703         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5704         [FW_DEVLOG_FACILITY_TM]         = "TM",
5705         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5706         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5707         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5708         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5709         [FW_DEVLOG_FACILITY_RI]         = "RI",
5710         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5711         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5712         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5713         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5714 };
5715
5716 static int
5717 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5718 {
5719         struct adapter *sc = arg1;
5720         struct devlog_params *dparams = &sc->params.devlog;
5721         struct fw_devlog_e *buf, *e;
5722         int i, j, rc, nentries, first = 0, m;
5723         struct sbuf *sb;
5724         uint64_t ftstamp = UINT64_MAX;
5725
5726         if (dparams->start == 0) {
5727                 dparams->memtype = FW_MEMTYPE_EDC0;
5728                 dparams->start = 0x84000;
5729                 dparams->size = 32768;
5730         }
5731
5732         nentries = dparams->size / sizeof(struct fw_devlog_e);
5733
5734         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5735         if (buf == NULL)
5736                 return (ENOMEM);
5737
5738         m = fwmtype_to_hwmtype(dparams->memtype);
5739         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5740         if (rc != 0)
5741                 goto done;
5742
5743         for (i = 0; i < nentries; i++) {
5744                 e = &buf[i];
5745
5746                 if (e->timestamp == 0)
5747                         break;  /* end */
5748
5749                 e->timestamp = be64toh(e->timestamp);
5750                 e->seqno = be32toh(e->seqno);
5751                 for (j = 0; j < 8; j++)
5752                         e->params[j] = be32toh(e->params[j]);
5753
5754                 if (e->timestamp < ftstamp) {
5755                         ftstamp = e->timestamp;
5756                         first = i;
5757                 }
5758         }
5759
5760         if (buf[first].timestamp == 0)
5761                 goto done;      /* nothing in the log */
5762
5763         rc = sysctl_wire_old_buffer(req, 0);
5764         if (rc != 0)
5765                 goto done;
5766
5767         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5768         if (sb == NULL) {
5769                 rc = ENOMEM;
5770                 goto done;
5771         }
5772         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5773             "Seq#", "Tstamp", "Level", "Facility", "Message");
5774
5775         i = first;
5776         do {
5777                 e = &buf[i];
5778                 if (e->timestamp == 0)
5779                         break;  /* end */
5780
5781                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5782                     e->seqno, e->timestamp,
5783                     (e->level < nitems(devlog_level_strings) ?
5784                         devlog_level_strings[e->level] : "UNKNOWN"),
5785                     (e->facility < nitems(devlog_facility_strings) ?
5786                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5787                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5788                     e->params[2], e->params[3], e->params[4],
5789                     e->params[5], e->params[6], e->params[7]);
5790
5791                 if (++i == nentries)
5792                         i = 0;
5793         } while (i != first);
5794
5795         rc = sbuf_finish(sb);
5796         sbuf_delete(sb);
5797 done:
5798         free(buf, M_CXGBE);
5799         return (rc);
5800 }
5801
5802 static int
5803 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5804 {
5805         struct adapter *sc = arg1;
5806         struct sbuf *sb;
5807         int rc;
5808         struct tp_fcoe_stats stats[4];
5809
5810         rc = sysctl_wire_old_buffer(req, 0);
5811         if (rc != 0)
5812                 return (rc);
5813
5814         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5815         if (sb == NULL)
5816                 return (ENOMEM);
5817
5818         t4_get_fcoe_stats(sc, 0, &stats[0]);
5819         t4_get_fcoe_stats(sc, 1, &stats[1]);
5820         t4_get_fcoe_stats(sc, 2, &stats[2]);
5821         t4_get_fcoe_stats(sc, 3, &stats[3]);
5822
5823         sbuf_printf(sb, "                   channel 0        channel 1        "
5824             "channel 2        channel 3\n");
5825         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5826             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5827             stats[3].octetsDDP);
5828         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5829             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5830         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5831             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5832             stats[3].framesDrop);
5833
5834         rc = sbuf_finish(sb);
5835         sbuf_delete(sb);
5836
5837         return (rc);
5838 }
5839
5840 static int
5841 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5842 {
5843         struct adapter *sc = arg1;
5844         struct sbuf *sb;
5845         int rc, i;
5846         unsigned int map, kbps, ipg, mode;
5847         unsigned int pace_tab[NTX_SCHED];
5848
5849         rc = sysctl_wire_old_buffer(req, 0);
5850         if (rc != 0)
5851                 return (rc);
5852
5853         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5854         if (sb == NULL)
5855                 return (ENOMEM);
5856
5857         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5858         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5859         t4_read_pace_tbl(sc, pace_tab);
5860
5861         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5862             "Class IPG (0.1 ns)   Flow IPG (us)");
5863
5864         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5865                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5866                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5867                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5868                 if (kbps)
5869                         sbuf_printf(sb, "%9u     ", kbps);
5870                 else
5871                         sbuf_printf(sb, " disabled     ");
5872
5873                 if (ipg)
5874                         sbuf_printf(sb, "%13u        ", ipg);
5875                 else
5876                         sbuf_printf(sb, "     disabled        ");
5877
5878                 if (pace_tab[i])
5879                         sbuf_printf(sb, "%10u", pace_tab[i]);
5880                 else
5881                         sbuf_printf(sb, "  disabled");
5882         }
5883
5884         rc = sbuf_finish(sb);
5885         sbuf_delete(sb);
5886
5887         return (rc);
5888 }
5889
5890 static int
5891 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5892 {
5893         struct adapter *sc = arg1;
5894         struct sbuf *sb;
5895         int rc, i, j;
5896         uint64_t *p0, *p1;
5897         struct lb_port_stats s[2];
5898         static const char *stat_name[] = {
5899                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5900                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5901                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5902                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5903                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5904                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5905                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5906         };
5907
5908         rc = sysctl_wire_old_buffer(req, 0);
5909         if (rc != 0)
5910                 return (rc);
5911
5912         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5913         if (sb == NULL)
5914                 return (ENOMEM);
5915
5916         memset(s, 0, sizeof(s));
5917
5918         for (i = 0; i < 4; i += 2) {
5919                 t4_get_lb_stats(sc, i, &s[0]);
5920                 t4_get_lb_stats(sc, i + 1, &s[1]);
5921
5922                 p0 = &s[0].octets;
5923                 p1 = &s[1].octets;
5924                 sbuf_printf(sb, "%s                       Loopback %u"
5925                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5926
5927                 for (j = 0; j < nitems(stat_name); j++)
5928                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5929                                    *p0++, *p1++);
5930         }
5931
5932         rc = sbuf_finish(sb);
5933         sbuf_delete(sb);
5934
5935         return (rc);
5936 }
5937
5938 static int
5939 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5940 {
5941         int rc = 0;
5942         struct port_info *pi = arg1;
5943         struct sbuf *sb;
5944         static const char *linkdnreasons[] = {
5945                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5946                 "PHY overheated", "unknown", "rx los", "reserved7"
5947         };
5948
5949         rc = sysctl_wire_old_buffer(req, 0);
5950         if (rc != 0)
5951                 return(rc);
5952         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5953         if (sb == NULL)
5954                 return (ENOMEM);
5955
5956         if (pi->linkdnrc < 0)
5957                 sbuf_printf(sb, "n/a");
5958         else if (pi->linkdnrc < nitems(linkdnreasons))
5959                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5960         else
5961                 sbuf_printf(sb, "%d", pi->linkdnrc);
5962
5963         rc = sbuf_finish(sb);
5964         sbuf_delete(sb);
5965
5966         return (rc);
5967 }
5968
5969 struct mem_desc {
5970         unsigned int base;
5971         unsigned int limit;
5972         unsigned int idx;
5973 };
5974
5975 static int
5976 mem_desc_cmp(const void *a, const void *b)
5977 {
5978         return ((const struct mem_desc *)a)->base -
5979                ((const struct mem_desc *)b)->base;
5980 }
5981
5982 static void
5983 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5984     unsigned int to)
5985 {
5986         unsigned int size;
5987
5988         size = to - from + 1;
5989         if (size == 0)
5990                 return;
5991
5992         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5993         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5994 }
5995
5996 static int
5997 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5998 {
5999         struct adapter *sc = arg1;
6000         struct sbuf *sb;
6001         int rc, i, n;
6002         uint32_t lo, hi, used, alloc;
6003         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6004         static const char *region[] = {
6005                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6006                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6007                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6008                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6009                 "RQUDP region:", "PBL region:", "TXPBL region:",
6010                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6011                 "On-chip queues:"
6012         };
6013         struct mem_desc avail[4];
6014         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6015         struct mem_desc *md = mem;
6016
6017         rc = sysctl_wire_old_buffer(req, 0);
6018         if (rc != 0)
6019                 return (rc);
6020
6021         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6022         if (sb == NULL)
6023                 return (ENOMEM);
6024
6025         for (i = 0; i < nitems(mem); i++) {
6026                 mem[i].limit = 0;
6027                 mem[i].idx = i;
6028         }
6029
6030         /* Find and sort the populated memory ranges */
6031         i = 0;
6032         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6033         if (lo & F_EDRAM0_ENABLE) {
6034                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6035                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6036                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6037                 avail[i].idx = 0;
6038                 i++;
6039         }
6040         if (lo & F_EDRAM1_ENABLE) {
6041                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6042                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6043                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6044                 avail[i].idx = 1;
6045                 i++;
6046         }
6047         if (lo & F_EXT_MEM_ENABLE) {
6048                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6049                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6050                 avail[i].limit = avail[i].base +
6051                     (G_EXT_MEM_SIZE(hi) << 20);
6052                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6053                 i++;
6054         }
6055         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6056                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6057                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6058                 avail[i].limit = avail[i].base +
6059                     (G_EXT_MEM1_SIZE(hi) << 20);
6060                 avail[i].idx = 4;
6061                 i++;
6062         }
6063         if (!i)                                    /* no memory available */
6064                 return 0;
6065         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6066
6067         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6068         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6069         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6070         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6071         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6072         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6073         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6074         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6075         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6076
6077         /* the next few have explicit upper bounds */
6078         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6079         md->limit = md->base - 1 +
6080                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6081                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6082         md++;
6083
6084         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6085         md->limit = md->base - 1 +
6086                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6087                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6088         md++;
6089
6090         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6091                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6092                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6093                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6094         } else {
6095                 md->base = 0;
6096                 md->idx = nitems(region);  /* hide it */
6097         }
6098         md++;
6099
6100 #define ulp_region(reg) \
6101         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6102         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6103
6104         ulp_region(RX_ISCSI);
6105         ulp_region(RX_TDDP);
6106         ulp_region(TX_TPT);
6107         ulp_region(RX_STAG);
6108         ulp_region(RX_RQ);
6109         ulp_region(RX_RQUDP);
6110         ulp_region(RX_PBL);
6111         ulp_region(TX_PBL);
6112 #undef ulp_region
6113
6114         md->base = 0;
6115         md->idx = nitems(region);
6116         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6117                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6118                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6119                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6120         }
6121         md++;
6122
6123         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6124         md->limit = md->base + sc->tids.ntids - 1;
6125         md++;
6126         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6127         md->limit = md->base + sc->tids.ntids - 1;
6128         md++;
6129
6130         md->base = sc->vres.ocq.start;
6131         if (sc->vres.ocq.size)
6132                 md->limit = md->base + sc->vres.ocq.size - 1;
6133         else
6134                 md->idx = nitems(region);  /* hide it */
6135         md++;
6136
6137         /* add any address-space holes, there can be up to 3 */
6138         for (n = 0; n < i - 1; n++)
6139                 if (avail[n].limit < avail[n + 1].base)
6140                         (md++)->base = avail[n].limit;
6141         if (avail[n].limit)
6142                 (md++)->base = avail[n].limit;
6143
6144         n = md - mem;
6145         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6146
6147         for (lo = 0; lo < i; lo++)
6148                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6149                                 avail[lo].limit - 1);
6150
6151         sbuf_printf(sb, "\n");
6152         for (i = 0; i < n; i++) {
6153                 if (mem[i].idx >= nitems(region))
6154                         continue;                        /* skip holes */
6155                 if (!mem[i].limit)
6156                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6157                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6158                                 mem[i].limit);
6159         }
6160
6161         sbuf_printf(sb, "\n");
6162         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6163         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6164         mem_region_show(sb, "uP RAM:", lo, hi);
6165
6166         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6167         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6168         mem_region_show(sb, "uP Extmem2:", lo, hi);
6169
6170         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6171         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6172                    G_PMRXMAXPAGE(lo),
6173                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6174                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6175
6176         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6177         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6178         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6179                    G_PMTXMAXPAGE(lo),
6180                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6181                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6182         sbuf_printf(sb, "%u p-structs\n",
6183                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6184
6185         for (i = 0; i < 4; i++) {
6186                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6187                 if (is_t4(sc)) {
6188                         used = G_USED(lo);
6189                         alloc = G_ALLOC(lo);
6190                 } else {
6191                         used = G_T5_USED(lo);
6192                         alloc = G_T5_ALLOC(lo);
6193                 }
6194                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6195                            i, used, alloc);
6196         }
6197         for (i = 0; i < 4; i++) {
6198                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6199                 if (is_t4(sc)) {
6200                         used = G_USED(lo);
6201                         alloc = G_ALLOC(lo);
6202                 } else {
6203                         used = G_T5_USED(lo);
6204                         alloc = G_T5_ALLOC(lo);
6205                 }
6206                 sbuf_printf(sb,
6207                            "\nLoopback %d using %u pages out of %u allocated",
6208                            i, used, alloc);
6209         }
6210
6211         rc = sbuf_finish(sb);
6212         sbuf_delete(sb);
6213
6214         return (rc);
6215 }
6216
6217 static inline void
6218 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6219 {
6220         *mask = x | y;
6221         y = htobe64(y);
6222         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6223 }
6224
6225 static int
6226 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6227 {
6228         struct adapter *sc = arg1;
6229         struct sbuf *sb;
6230         int rc, i, n;
6231
6232         rc = sysctl_wire_old_buffer(req, 0);
6233         if (rc != 0)
6234                 return (rc);
6235
6236         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6237         if (sb == NULL)
6238                 return (ENOMEM);
6239
6240         sbuf_printf(sb,
6241             "Idx  Ethernet address     Mask     Vld Ports PF"
6242             "  VF              Replication             P0 P1 P2 P3  ML");
6243         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6244             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6245         for (i = 0; i < n; i++) {
6246                 uint64_t tcamx, tcamy, mask;
6247                 uint32_t cls_lo, cls_hi;
6248                 uint8_t addr[ETHER_ADDR_LEN];
6249
6250                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6251                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6252                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6253                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6254
6255                 if (tcamx & tcamy)
6256                         continue;
6257
6258                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6259                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6260                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6261                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6262                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6263                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6264                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6265
6266                 if (cls_lo & F_REPLICATE) {
6267                         struct fw_ldst_cmd ldst_cmd;
6268
6269                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6270                         ldst_cmd.op_to_addrspace =
6271                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6272                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6273                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6274                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6275                         ldst_cmd.u.mps.fid_ctl =
6276                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6277                                 V_FW_LDST_CMD_CTL(i));
6278
6279                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6280                             "t4mps");
6281                         if (rc)
6282                                 break;
6283                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6284                             sizeof(ldst_cmd), &ldst_cmd);
6285                         end_synchronized_op(sc, 0);
6286
6287                         if (rc != 0) {
6288                                 sbuf_printf(sb,
6289                                     " ------------ error %3u ------------", rc);
6290                                 rc = 0;
6291                         } else {
6292                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6293                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6294                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6295                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6296                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6297                         }
6298                 } else
6299                         sbuf_printf(sb, "%36s", "");
6300
6301                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6302                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6303                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6304         }
6305
6306         if (rc)
6307                 (void) sbuf_finish(sb);
6308         else
6309                 rc = sbuf_finish(sb);
6310         sbuf_delete(sb);
6311
6312         return (rc);
6313 }
6314
6315 static int
6316 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6317 {
6318         struct adapter *sc = arg1;
6319         struct sbuf *sb;
6320         int rc;
6321         uint16_t mtus[NMTUS];
6322
6323         rc = sysctl_wire_old_buffer(req, 0);
6324         if (rc != 0)
6325                 return (rc);
6326
6327         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6328         if (sb == NULL)
6329                 return (ENOMEM);
6330
6331         t4_read_mtu_tbl(sc, mtus, NULL);
6332
6333         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6334             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6335             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6336             mtus[14], mtus[15]);
6337
6338         rc = sbuf_finish(sb);
6339         sbuf_delete(sb);
6340
6341         return (rc);
6342 }
6343
6344 static int
6345 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6346 {
6347         struct adapter *sc = arg1;
6348         struct sbuf *sb;
6349         int rc, i;
6350         uint32_t cnt[PM_NSTATS];
6351         uint64_t cyc[PM_NSTATS];
6352         static const char *rx_stats[] = {
6353                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6354         };
6355         static const char *tx_stats[] = {
6356                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6357         };
6358
6359         rc = sysctl_wire_old_buffer(req, 0);
6360         if (rc != 0)
6361                 return (rc);
6362
6363         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6364         if (sb == NULL)
6365                 return (ENOMEM);
6366
6367         t4_pmtx_get_stats(sc, cnt, cyc);
6368         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6369         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6370                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6371                     cyc[i]);
6372
6373         t4_pmrx_get_stats(sc, cnt, cyc);
6374         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6375         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6376                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6377                     cyc[i]);
6378
6379         rc = sbuf_finish(sb);
6380         sbuf_delete(sb);
6381
6382         return (rc);
6383 }
6384
6385 static int
6386 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6387 {
6388         struct adapter *sc = arg1;
6389         struct sbuf *sb;
6390         int rc;
6391         struct tp_rdma_stats stats;
6392
6393         rc = sysctl_wire_old_buffer(req, 0);
6394         if (rc != 0)
6395                 return (rc);
6396
6397         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6398         if (sb == NULL)
6399                 return (ENOMEM);
6400
6401         t4_tp_get_rdma_stats(sc, &stats);
6402         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6403         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6404
6405         rc = sbuf_finish(sb);
6406         sbuf_delete(sb);
6407
6408         return (rc);
6409 }
6410
6411 static int
6412 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6413 {
6414         struct adapter *sc = arg1;
6415         struct sbuf *sb;
6416         int rc;
6417         struct tp_tcp_stats v4, v6;
6418
6419         rc = sysctl_wire_old_buffer(req, 0);
6420         if (rc != 0)
6421                 return (rc);
6422
6423         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6424         if (sb == NULL)
6425                 return (ENOMEM);
6426
6427         t4_tp_get_tcp_stats(sc, &v4, &v6);
6428         sbuf_printf(sb,
6429             "                                IP                 IPv6\n");
6430         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6431             v4.tcpOutRsts, v6.tcpOutRsts);
6432         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6433             v4.tcpInSegs, v6.tcpInSegs);
6434         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6435             v4.tcpOutSegs, v6.tcpOutSegs);
6436         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6437             v4.tcpRetransSegs, v6.tcpRetransSegs);
6438
6439         rc = sbuf_finish(sb);
6440         sbuf_delete(sb);
6441
6442         return (rc);
6443 }
6444
6445 static int
6446 sysctl_tids(SYSCTL_HANDLER_ARGS)
6447 {
6448         struct adapter *sc = arg1;
6449         struct sbuf *sb;
6450         int rc;
6451         struct tid_info *t = &sc->tids;
6452
6453         rc = sysctl_wire_old_buffer(req, 0);
6454         if (rc != 0)
6455                 return (rc);
6456
6457         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6458         if (sb == NULL)
6459                 return (ENOMEM);
6460
6461         if (t->natids) {
6462                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6463                     t->atids_in_use);
6464         }
6465
6466         if (t->ntids) {
6467                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6468                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6469
6470                         if (b) {
6471                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6472                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6473                                     t->ntids - 1);
6474                         } else {
6475                                 sbuf_printf(sb, "TID range: %u-%u",
6476                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6477                                     t->ntids - 1);
6478                         }
6479                 } else
6480                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6481                 sbuf_printf(sb, ", in use: %u\n",
6482                     atomic_load_acq_int(&t->tids_in_use));
6483         }
6484
6485         if (t->nstids) {
6486                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6487                     t->stid_base + t->nstids - 1, t->stids_in_use);
6488         }
6489
6490         if (t->nftids) {
6491                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6492                     t->ftid_base + t->nftids - 1);
6493         }
6494
6495         if (t->netids) {
6496                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6497                     t->etid_base + t->netids - 1);
6498         }
6499
6500         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6501             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6502             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6503
6504         rc = sbuf_finish(sb);
6505         sbuf_delete(sb);
6506
6507         return (rc);
6508 }
6509
6510 static int
6511 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6512 {
6513         struct adapter *sc = arg1;
6514         struct sbuf *sb;
6515         int rc;
6516         struct tp_err_stats stats;
6517
6518         rc = sysctl_wire_old_buffer(req, 0);
6519         if (rc != 0)
6520                 return (rc);
6521
6522         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6523         if (sb == NULL)
6524                 return (ENOMEM);
6525
6526         t4_tp_get_err_stats(sc, &stats);
6527
6528         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6529                       "channel 3\n");
6530         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6531             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6532             stats.macInErrs[3]);
6533         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6534             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6535             stats.hdrInErrs[3]);
6536         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6537             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6538             stats.tcpInErrs[3]);
6539         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6540             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6541             stats.tcp6InErrs[3]);
6542         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6543             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6544             stats.tnlCongDrops[3]);
6545         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6546             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6547             stats.tnlTxDrops[3]);
6548         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6549             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6550             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6551         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6552             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6553             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6554         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6555             stats.ofldNoNeigh, stats.ofldCongDefer);
6556
6557         rc = sbuf_finish(sb);
6558         sbuf_delete(sb);
6559
6560         return (rc);
6561 }
6562
6563 struct field_desc {
6564         const char *name;
6565         u_int start;
6566         u_int width;
6567 };
6568
6569 static void
6570 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6571 {
6572         char buf[32];
6573         int line_size = 0;
6574
6575         while (f->name) {
6576                 uint64_t mask = (1ULL << f->width) - 1;
6577                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6578                     ((uintmax_t)v >> f->start) & mask);
6579
6580                 if (line_size + len >= 79) {
6581                         line_size = 8;
6582                         sbuf_printf(sb, "\n        ");
6583                 }
6584                 sbuf_printf(sb, "%s ", buf);
6585                 line_size += len + 1;
6586                 f++;
6587         }
6588         sbuf_printf(sb, "\n");
6589 }
6590
6591 static struct field_desc tp_la0[] = {
6592         { "RcfOpCodeOut", 60, 4 },
6593         { "State", 56, 4 },
6594         { "WcfState", 52, 4 },
6595         { "RcfOpcSrcOut", 50, 2 },
6596         { "CRxError", 49, 1 },
6597         { "ERxError", 48, 1 },
6598         { "SanityFailed", 47, 1 },
6599         { "SpuriousMsg", 46, 1 },
6600         { "FlushInputMsg", 45, 1 },
6601         { "FlushInputCpl", 44, 1 },
6602         { "RssUpBit", 43, 1 },
6603         { "RssFilterHit", 42, 1 },
6604         { "Tid", 32, 10 },
6605         { "InitTcb", 31, 1 },
6606         { "LineNumber", 24, 7 },
6607         { "Emsg", 23, 1 },
6608         { "EdataOut", 22, 1 },
6609         { "Cmsg", 21, 1 },
6610         { "CdataOut", 20, 1 },
6611         { "EreadPdu", 19, 1 },
6612         { "CreadPdu", 18, 1 },
6613         { "TunnelPkt", 17, 1 },
6614         { "RcfPeerFin", 16, 1 },
6615         { "RcfReasonOut", 12, 4 },
6616         { "TxCchannel", 10, 2 },
6617         { "RcfTxChannel", 8, 2 },
6618         { "RxEchannel", 6, 2 },
6619         { "RcfRxChannel", 5, 1 },
6620         { "RcfDataOutSrdy", 4, 1 },
6621         { "RxDvld", 3, 1 },
6622         { "RxOoDvld", 2, 1 },
6623         { "RxCongestion", 1, 1 },
6624         { "TxCongestion", 0, 1 },
6625         { NULL }
6626 };
6627
6628 static struct field_desc tp_la1[] = {
6629         { "CplCmdIn", 56, 8 },
6630         { "CplCmdOut", 48, 8 },
6631         { "ESynOut", 47, 1 },
6632         { "EAckOut", 46, 1 },
6633         { "EFinOut", 45, 1 },
6634         { "ERstOut", 44, 1 },
6635         { "SynIn", 43, 1 },
6636         { "AckIn", 42, 1 },
6637         { "FinIn", 41, 1 },
6638         { "RstIn", 40, 1 },
6639         { "DataIn", 39, 1 },
6640         { "DataInVld", 38, 1 },
6641         { "PadIn", 37, 1 },
6642         { "RxBufEmpty", 36, 1 },
6643         { "RxDdp", 35, 1 },
6644         { "RxFbCongestion", 34, 1 },
6645         { "TxFbCongestion", 33, 1 },
6646         { "TxPktSumSrdy", 32, 1 },
6647         { "RcfUlpType", 28, 4 },
6648         { "Eread", 27, 1 },
6649         { "Ebypass", 26, 1 },
6650         { "Esave", 25, 1 },
6651         { "Static0", 24, 1 },
6652         { "Cread", 23, 1 },
6653         { "Cbypass", 22, 1 },
6654         { "Csave", 21, 1 },
6655         { "CPktOut", 20, 1 },
6656         { "RxPagePoolFull", 18, 2 },
6657         { "RxLpbkPkt", 17, 1 },
6658         { "TxLpbkPkt", 16, 1 },
6659         { "RxVfValid", 15, 1 },
6660         { "SynLearned", 14, 1 },
6661         { "SetDelEntry", 13, 1 },
6662         { "SetInvEntry", 12, 1 },
6663         { "CpcmdDvld", 11, 1 },
6664         { "CpcmdSave", 10, 1 },
6665         { "RxPstructsFull", 8, 2 },
6666         { "EpcmdDvld", 7, 1 },
6667         { "EpcmdFlush", 6, 1 },
6668         { "EpcmdTrimPrefix", 5, 1 },
6669         { "EpcmdTrimPostfix", 4, 1 },
6670         { "ERssIp4Pkt", 3, 1 },
6671         { "ERssIp6Pkt", 2, 1 },
6672         { "ERssTcpUdpPkt", 1, 1 },
6673         { "ERssFceFipPkt", 0, 1 },
6674         { NULL }
6675 };
6676
6677 static struct field_desc tp_la2[] = {
6678         { "CplCmdIn", 56, 8 },
6679         { "MpsVfVld", 55, 1 },
6680         { "MpsPf", 52, 3 },
6681         { "MpsVf", 44, 8 },
6682         { "SynIn", 43, 1 },
6683         { "AckIn", 42, 1 },
6684         { "FinIn", 41, 1 },
6685         { "RstIn", 40, 1 },
6686         { "DataIn", 39, 1 },
6687         { "DataInVld", 38, 1 },
6688         { "PadIn", 37, 1 },
6689         { "RxBufEmpty", 36, 1 },
6690         { "RxDdp", 35, 1 },
6691         { "RxFbCongestion", 34, 1 },
6692         { "TxFbCongestion", 33, 1 },
6693         { "TxPktSumSrdy", 32, 1 },
6694         { "RcfUlpType", 28, 4 },
6695         { "Eread", 27, 1 },
6696         { "Ebypass", 26, 1 },
6697         { "Esave", 25, 1 },
6698         { "Static0", 24, 1 },
6699         { "Cread", 23, 1 },
6700         { "Cbypass", 22, 1 },
6701         { "Csave", 21, 1 },
6702         { "CPktOut", 20, 1 },
6703         { "RxPagePoolFull", 18, 2 },
6704         { "RxLpbkPkt", 17, 1 },
6705         { "TxLpbkPkt", 16, 1 },
6706         { "RxVfValid", 15, 1 },
6707         { "SynLearned", 14, 1 },
6708         { "SetDelEntry", 13, 1 },
6709         { "SetInvEntry", 12, 1 },
6710         { "CpcmdDvld", 11, 1 },
6711         { "CpcmdSave", 10, 1 },
6712         { "RxPstructsFull", 8, 2 },
6713         { "EpcmdDvld", 7, 1 },
6714         { "EpcmdFlush", 6, 1 },
6715         { "EpcmdTrimPrefix", 5, 1 },
6716         { "EpcmdTrimPostfix", 4, 1 },
6717         { "ERssIp4Pkt", 3, 1 },
6718         { "ERssIp6Pkt", 2, 1 },
6719         { "ERssTcpUdpPkt", 1, 1 },
6720         { "ERssFceFipPkt", 0, 1 },
6721         { NULL }
6722 };
6723
6724 static void
6725 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6726 {
6727
6728         field_desc_show(sb, *p, tp_la0);
6729 }
6730
6731 static void
6732 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6733 {
6734
6735         if (idx)
6736                 sbuf_printf(sb, "\n");
6737         field_desc_show(sb, p[0], tp_la0);
6738         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6739                 field_desc_show(sb, p[1], tp_la0);
6740 }
6741
6742 static void
6743 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6744 {
6745
6746         if (idx)
6747                 sbuf_printf(sb, "\n");
6748         field_desc_show(sb, p[0], tp_la0);
6749         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6750                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6751 }
6752
6753 static int
6754 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6755 {
6756         struct adapter *sc = arg1;
6757         struct sbuf *sb;
6758         uint64_t *buf, *p;
6759         int rc;
6760         u_int i, inc;
6761         void (*show_func)(struct sbuf *, uint64_t *, int);
6762
6763         rc = sysctl_wire_old_buffer(req, 0);
6764         if (rc != 0)
6765                 return (rc);
6766
6767         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6768         if (sb == NULL)
6769                 return (ENOMEM);
6770
6771         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6772
6773         t4_tp_read_la(sc, buf, NULL);
6774         p = buf;
6775
6776         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6777         case 2:
6778                 inc = 2;
6779                 show_func = tp_la_show2;
6780                 break;
6781         case 3:
6782                 inc = 2;
6783                 show_func = tp_la_show3;
6784                 break;
6785         default:
6786                 inc = 1;
6787                 show_func = tp_la_show;
6788         }
6789
6790         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6791                 (*show_func)(sb, p, i);
6792
6793         rc = sbuf_finish(sb);
6794         sbuf_delete(sb);
6795         free(buf, M_CXGBE);
6796         return (rc);
6797 }
6798
6799 static int
6800 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6801 {
6802         struct adapter *sc = arg1;
6803         struct sbuf *sb;
6804         int rc;
6805         u64 nrate[NCHAN], orate[NCHAN];
6806
6807         rc = sysctl_wire_old_buffer(req, 0);
6808         if (rc != 0)
6809                 return (rc);
6810
6811         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6812         if (sb == NULL)
6813                 return (ENOMEM);
6814
6815         t4_get_chan_txrate(sc, nrate, orate);
6816         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6817                  "channel 3\n");
6818         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6819             nrate[0], nrate[1], nrate[2], nrate[3]);
6820         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6821             orate[0], orate[1], orate[2], orate[3]);
6822
6823         rc = sbuf_finish(sb);
6824         sbuf_delete(sb);
6825
6826         return (rc);
6827 }
6828
6829 static int
6830 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6831 {
6832         struct adapter *sc = arg1;
6833         struct sbuf *sb;
6834         uint32_t *buf, *p;
6835         int rc, i;
6836
6837         rc = sysctl_wire_old_buffer(req, 0);
6838         if (rc != 0)
6839                 return (rc);
6840
6841         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6842         if (sb == NULL)
6843                 return (ENOMEM);
6844
6845         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6846             M_ZERO | M_WAITOK);
6847
6848         t4_ulprx_read_la(sc, buf);
6849         p = buf;
6850
6851         sbuf_printf(sb, "      Pcmd        Type   Message"
6852             "                Data");
6853         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6854                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6855                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6856         }
6857
6858         rc = sbuf_finish(sb);
6859         sbuf_delete(sb);
6860         free(buf, M_CXGBE);
6861         return (rc);
6862 }
6863
6864 static int
6865 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6866 {
6867         struct adapter *sc = arg1;
6868         struct sbuf *sb;
6869         int rc, v;
6870
6871         rc = sysctl_wire_old_buffer(req, 0);
6872         if (rc != 0)
6873                 return (rc);
6874
6875         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6876         if (sb == NULL)
6877                 return (ENOMEM);
6878
6879         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6880         if (G_STATSOURCE_T5(v) == 7) {
6881                 if (G_STATMODE(v) == 0) {
6882                         sbuf_printf(sb, "total %d, incomplete %d",
6883                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6884                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6885                 } else if (G_STATMODE(v) == 1) {
6886                         sbuf_printf(sb, "total %d, data overflow %d",
6887                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6888                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6889                 }
6890         }
6891         rc = sbuf_finish(sb);
6892         sbuf_delete(sb);
6893
6894         return (rc);
6895 }
6896 #endif
6897
6898 static uint32_t
6899 fconf_to_mode(uint32_t fconf)
6900 {
6901         uint32_t mode;
6902
6903         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6904             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6905
6906         if (fconf & F_FRAGMENTATION)
6907                 mode |= T4_FILTER_IP_FRAGMENT;
6908
6909         if (fconf & F_MPSHITTYPE)
6910                 mode |= T4_FILTER_MPS_HIT_TYPE;
6911
6912         if (fconf & F_MACMATCH)
6913                 mode |= T4_FILTER_MAC_IDX;
6914
6915         if (fconf & F_ETHERTYPE)
6916                 mode |= T4_FILTER_ETH_TYPE;
6917
6918         if (fconf & F_PROTOCOL)
6919                 mode |= T4_FILTER_IP_PROTO;
6920
6921         if (fconf & F_TOS)
6922                 mode |= T4_FILTER_IP_TOS;
6923
6924         if (fconf & F_VLAN)
6925                 mode |= T4_FILTER_VLAN;
6926
6927         if (fconf & F_VNIC_ID)
6928                 mode |= T4_FILTER_VNIC;
6929
6930         if (fconf & F_PORT)
6931                 mode |= T4_FILTER_PORT;
6932
6933         if (fconf & F_FCOE)
6934                 mode |= T4_FILTER_FCoE;
6935
6936         return (mode);
6937 }
6938
6939 static uint32_t
6940 mode_to_fconf(uint32_t mode)
6941 {
6942         uint32_t fconf = 0;
6943
6944         if (mode & T4_FILTER_IP_FRAGMENT)
6945                 fconf |= F_FRAGMENTATION;
6946
6947         if (mode & T4_FILTER_MPS_HIT_TYPE)
6948                 fconf |= F_MPSHITTYPE;
6949
6950         if (mode & T4_FILTER_MAC_IDX)
6951                 fconf |= F_MACMATCH;
6952
6953         if (mode & T4_FILTER_ETH_TYPE)
6954                 fconf |= F_ETHERTYPE;
6955
6956         if (mode & T4_FILTER_IP_PROTO)
6957                 fconf |= F_PROTOCOL;
6958
6959         if (mode & T4_FILTER_IP_TOS)
6960                 fconf |= F_TOS;
6961
6962         if (mode & T4_FILTER_VLAN)
6963                 fconf |= F_VLAN;
6964
6965         if (mode & T4_FILTER_VNIC)
6966                 fconf |= F_VNIC_ID;
6967
6968         if (mode & T4_FILTER_PORT)
6969                 fconf |= F_PORT;
6970
6971         if (mode & T4_FILTER_FCoE)
6972                 fconf |= F_FCOE;
6973
6974         return (fconf);
6975 }
6976
6977 static uint32_t
6978 fspec_to_fconf(struct t4_filter_specification *fs)
6979 {
6980         uint32_t fconf = 0;
6981
6982         if (fs->val.frag || fs->mask.frag)
6983                 fconf |= F_FRAGMENTATION;
6984
6985         if (fs->val.matchtype || fs->mask.matchtype)
6986                 fconf |= F_MPSHITTYPE;
6987
6988         if (fs->val.macidx || fs->mask.macidx)
6989                 fconf |= F_MACMATCH;
6990
6991         if (fs->val.ethtype || fs->mask.ethtype)
6992                 fconf |= F_ETHERTYPE;
6993
6994         if (fs->val.proto || fs->mask.proto)
6995                 fconf |= F_PROTOCOL;
6996
6997         if (fs->val.tos || fs->mask.tos)
6998                 fconf |= F_TOS;
6999
7000         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7001                 fconf |= F_VLAN;
7002
7003         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7004                 fconf |= F_VNIC_ID;
7005
7006         if (fs->val.iport || fs->mask.iport)
7007                 fconf |= F_PORT;
7008
7009         if (fs->val.fcoe || fs->mask.fcoe)
7010                 fconf |= F_FCOE;
7011
7012         return (fconf);
7013 }
7014
7015 static int
7016 get_filter_mode(struct adapter *sc, uint32_t *mode)
7017 {
7018         int rc;
7019         uint32_t fconf;
7020
7021         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7022             "t4getfm");
7023         if (rc)
7024                 return (rc);
7025
7026         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7027             A_TP_VLAN_PRI_MAP);
7028
7029         if (sc->params.tp.vlan_pri_map != fconf) {
7030                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7031                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7032                     fconf);
7033         }
7034
7035         *mode = fconf_to_mode(fconf);
7036
7037         end_synchronized_op(sc, LOCK_HELD);
7038         return (0);
7039 }
7040
7041 static int
7042 set_filter_mode(struct adapter *sc, uint32_t mode)
7043 {
7044         uint32_t fconf;
7045         int rc;
7046
7047         fconf = mode_to_fconf(mode);
7048
7049         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7050             "t4setfm");
7051         if (rc)
7052                 return (rc);
7053
7054         if (sc->tids.ftids_in_use > 0) {
7055                 rc = EBUSY;
7056                 goto done;
7057         }
7058
7059 #ifdef TCP_OFFLOAD
7060         if (uld_active(sc, ULD_TOM)) {
7061                 rc = EBUSY;
7062                 goto done;
7063         }
7064 #endif
7065
7066         rc = -t4_set_filter_mode(sc, fconf);
7067 done:
7068         end_synchronized_op(sc, LOCK_HELD);
7069         return (rc);
7070 }
7071
7072 static inline uint64_t
7073 get_filter_hits(struct adapter *sc, uint32_t fid)
7074 {
7075         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7076         uint64_t hits;
7077
7078         memwin_info(sc, 0, &mw_base, NULL);
7079         off = position_memwin(sc, 0,
7080             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7081         if (is_t4(sc)) {
7082                 hits = t4_read_reg64(sc, mw_base + off + 16);
7083                 hits = be64toh(hits);
7084         } else {
7085                 hits = t4_read_reg(sc, mw_base + off + 24);
7086                 hits = be32toh(hits);
7087         }
7088
7089         return (hits);
7090 }
7091
7092 static int
7093 get_filter(struct adapter *sc, struct t4_filter *t)
7094 {
7095         int i, rc, nfilters = sc->tids.nftids;
7096         struct filter_entry *f;
7097
7098         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7099             "t4getf");
7100         if (rc)
7101                 return (rc);
7102
7103         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7104             t->idx >= nfilters) {
7105                 t->idx = 0xffffffff;
7106                 goto done;
7107         }
7108
7109         f = &sc->tids.ftid_tab[t->idx];
7110         for (i = t->idx; i < nfilters; i++, f++) {
7111                 if (f->valid) {
7112                         t->idx = i;
7113                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7114                         t->smtidx = f->smtidx;
7115                         if (f->fs.hitcnts)
7116                                 t->hits = get_filter_hits(sc, t->idx);
7117                         else
7118                                 t->hits = UINT64_MAX;
7119                         t->fs = f->fs;
7120
7121                         goto done;
7122                 }
7123         }
7124
7125         t->idx = 0xffffffff;
7126 done:
7127         end_synchronized_op(sc, LOCK_HELD);
7128         return (0);
7129 }
7130
7131 static int
7132 set_filter(struct adapter *sc, struct t4_filter *t)
7133 {
7134         unsigned int nfilters, nports;
7135         struct filter_entry *f;
7136         int i, rc;
7137
7138         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7139         if (rc)
7140                 return (rc);
7141
7142         nfilters = sc->tids.nftids;
7143         nports = sc->params.nports;
7144
7145         if (nfilters == 0) {
7146                 rc = ENOTSUP;
7147                 goto done;
7148         }
7149
7150         if (!(sc->flags & FULL_INIT_DONE)) {
7151                 rc = EAGAIN;
7152                 goto done;
7153         }
7154
7155         if (t->idx >= nfilters) {
7156                 rc = EINVAL;
7157                 goto done;
7158         }
7159
7160         /* Validate against the global filter mode */
7161         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7162             sc->params.tp.vlan_pri_map) {
7163                 rc = E2BIG;
7164                 goto done;
7165         }
7166
7167         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7168                 rc = EINVAL;
7169                 goto done;
7170         }
7171
7172         if (t->fs.val.iport >= nports) {
7173                 rc = EINVAL;
7174                 goto done;
7175         }
7176
7177         /* Can't specify an iq if not steering to it */
7178         if (!t->fs.dirsteer && t->fs.iq) {
7179                 rc = EINVAL;
7180                 goto done;
7181         }
7182
7183         /* IPv6 filter idx must be 4 aligned */
7184         if (t->fs.type == 1 &&
7185             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7186                 rc = EINVAL;
7187                 goto done;
7188         }
7189
7190         if (sc->tids.ftid_tab == NULL) {
7191                 KASSERT(sc->tids.ftids_in_use == 0,
7192                     ("%s: no memory allocated but filters_in_use > 0",
7193                     __func__));
7194
7195                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7196                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7197                 if (sc->tids.ftid_tab == NULL) {
7198                         rc = ENOMEM;
7199                         goto done;
7200                 }
7201                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7202         }
7203
7204         for (i = 0; i < 4; i++) {
7205                 f = &sc->tids.ftid_tab[t->idx + i];
7206
7207                 if (f->pending || f->valid) {
7208                         rc = EBUSY;
7209                         goto done;
7210                 }
7211                 if (f->locked) {
7212                         rc = EPERM;
7213                         goto done;
7214                 }
7215
7216                 if (t->fs.type == 0)
7217                         break;
7218         }
7219
7220         f = &sc->tids.ftid_tab[t->idx];
7221         f->fs = t->fs;
7222
7223         rc = set_filter_wr(sc, t->idx);
7224 done:
7225         end_synchronized_op(sc, 0);
7226
7227         if (rc == 0) {
7228                 mtx_lock(&sc->tids.ftid_lock);
7229                 for (;;) {
7230                         if (f->pending == 0) {
7231                                 rc = f->valid ? 0 : EIO;
7232                                 break;
7233                         }
7234
7235                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7236                             PCATCH, "t4setfw", 0)) {
7237                                 rc = EINPROGRESS;
7238                                 break;
7239                         }
7240                 }
7241                 mtx_unlock(&sc->tids.ftid_lock);
7242         }
7243         return (rc);
7244 }
7245
7246 static int
7247 del_filter(struct adapter *sc, struct t4_filter *t)
7248 {
7249         unsigned int nfilters;
7250         struct filter_entry *f;
7251         int rc;
7252
7253         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7254         if (rc)
7255                 return (rc);
7256
7257         nfilters = sc->tids.nftids;
7258
7259         if (nfilters == 0) {
7260                 rc = ENOTSUP;
7261                 goto done;
7262         }
7263
7264         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7265             t->idx >= nfilters) {
7266                 rc = EINVAL;
7267                 goto done;
7268         }
7269
7270         if (!(sc->flags & FULL_INIT_DONE)) {
7271                 rc = EAGAIN;
7272                 goto done;
7273         }
7274
7275         f = &sc->tids.ftid_tab[t->idx];
7276
7277         if (f->pending) {
7278                 rc = EBUSY;
7279                 goto done;
7280         }
7281         if (f->locked) {
7282                 rc = EPERM;
7283                 goto done;
7284         }
7285
7286         if (f->valid) {
7287                 t->fs = f->fs;  /* extra info for the caller */
7288                 rc = del_filter_wr(sc, t->idx);
7289         }
7290
7291 done:
7292         end_synchronized_op(sc, 0);
7293
7294         if (rc == 0) {
7295                 mtx_lock(&sc->tids.ftid_lock);
7296                 for (;;) {
7297                         if (f->pending == 0) {
7298                                 rc = f->valid ? EIO : 0;
7299                                 break;
7300                         }
7301
7302                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7303                             PCATCH, "t4delfw", 0)) {
7304                                 rc = EINPROGRESS;
7305                                 break;
7306                         }
7307                 }
7308                 mtx_unlock(&sc->tids.ftid_lock);
7309         }
7310
7311         return (rc);
7312 }
7313
7314 static void
7315 clear_filter(struct filter_entry *f)
7316 {
7317         if (f->l2t)
7318                 t4_l2t_release(f->l2t);
7319
7320         bzero(f, sizeof (*f));
7321 }
7322
7323 static int
7324 set_filter_wr(struct adapter *sc, int fidx)
7325 {
7326         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7327         struct fw_filter_wr *fwr;
7328         unsigned int ftid;
7329         struct wrq_cookie cookie;
7330
7331         ASSERT_SYNCHRONIZED_OP(sc);
7332
7333         if (f->fs.newdmac || f->fs.newvlan) {
7334                 /* This filter needs an L2T entry; allocate one. */
7335                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7336                 if (f->l2t == NULL)
7337                         return (EAGAIN);
7338                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7339                     f->fs.dmac)) {
7340                         t4_l2t_release(f->l2t);
7341                         f->l2t = NULL;
7342                         return (ENOMEM);
7343                 }
7344         }
7345
7346         ftid = sc->tids.ftid_base + fidx;
7347
7348         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7349         if (fwr == NULL)
7350                 return (ENOMEM);
7351         bzero(fwr, sizeof(*fwr));
7352
7353         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7354         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7355         fwr->tid_to_iq =
7356             htobe32(V_FW_FILTER_WR_TID(ftid) |
7357                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7358                 V_FW_FILTER_WR_NOREPLY(0) |
7359                 V_FW_FILTER_WR_IQ(f->fs.iq));
7360         fwr->del_filter_to_l2tix =
7361             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7362                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7363                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7364                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7365                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7366                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7367                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7368                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7369                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7370                     f->fs.newvlan == VLAN_REWRITE) |
7371                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7372                     f->fs.newvlan == VLAN_REWRITE) |
7373                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7374                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7375                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7376                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7377         fwr->ethtype = htobe16(f->fs.val.ethtype);
7378         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7379         fwr->frag_to_ovlan_vldm =
7380             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7381                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7382                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7383                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7384                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7385                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7386         fwr->smac_sel = 0;
7387         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7388             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7389         fwr->maci_to_matchtypem =
7390             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7391                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7392                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7393                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7394                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7395                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7396                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7397                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7398         fwr->ptcl = f->fs.val.proto;
7399         fwr->ptclm = f->fs.mask.proto;
7400         fwr->ttyp = f->fs.val.tos;
7401         fwr->ttypm = f->fs.mask.tos;
7402         fwr->ivlan = htobe16(f->fs.val.vlan);
7403         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7404         fwr->ovlan = htobe16(f->fs.val.vnic);
7405         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7406         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7407         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7408         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7409         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7410         fwr->lp = htobe16(f->fs.val.dport);
7411         fwr->lpm = htobe16(f->fs.mask.dport);
7412         fwr->fp = htobe16(f->fs.val.sport);
7413         fwr->fpm = htobe16(f->fs.mask.sport);
7414         if (f->fs.newsmac)
7415                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7416
7417         f->pending = 1;
7418         sc->tids.ftids_in_use++;
7419
7420         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7421         return (0);
7422 }
7423
7424 static int
7425 del_filter_wr(struct adapter *sc, int fidx)
7426 {
7427         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7428         struct fw_filter_wr *fwr;
7429         unsigned int ftid;
7430         struct wrq_cookie cookie;
7431
7432         ftid = sc->tids.ftid_base + fidx;
7433
7434         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7435         if (fwr == NULL)
7436                 return (ENOMEM);
7437         bzero(fwr, sizeof (*fwr));
7438
7439         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7440
7441         f->pending = 1;
7442         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7443         return (0);
7444 }
7445
7446 int
7447 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7448 {
7449         struct adapter *sc = iq->adapter;
7450         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7451         unsigned int idx = GET_TID(rpl);
7452         unsigned int rc;
7453         struct filter_entry *f;
7454
7455         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7456             rss->opcode));
7457
7458         if (is_ftid(sc, idx)) {
7459
7460                 idx -= sc->tids.ftid_base;
7461                 f = &sc->tids.ftid_tab[idx];
7462                 rc = G_COOKIE(rpl->cookie);
7463
7464                 mtx_lock(&sc->tids.ftid_lock);
7465                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7466                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7467                             __func__, idx));
7468                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7469                         f->pending = 0;  /* asynchronous setup completed */
7470                         f->valid = 1;
7471                 } else {
7472                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7473                                 /* Add or delete failed, display an error */
7474                                 log(LOG_ERR,
7475                                     "filter %u setup failed with error %u\n",
7476                                     idx, rc);
7477                         }
7478
7479                         clear_filter(f);
7480                         sc->tids.ftids_in_use--;
7481                 }
7482                 wakeup(&sc->tids.ftid_tab);
7483                 mtx_unlock(&sc->tids.ftid_lock);
7484         }
7485
7486         return (0);
7487 }
7488
7489 static int
7490 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7491 {
7492         int rc;
7493
7494         if (cntxt->cid > M_CTXTQID)
7495                 return (EINVAL);
7496
7497         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7498             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7499                 return (EINVAL);
7500
7501         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7502         if (rc)
7503                 return (rc);
7504
7505         if (sc->flags & FW_OK) {
7506                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7507                     &cntxt->data[0]);
7508                 if (rc == 0)
7509                         goto done;
7510         }
7511
7512         /*
7513          * Read via firmware failed or wasn't even attempted.  Read directly via
7514          * the backdoor.
7515          */
7516         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7517 done:
7518         end_synchronized_op(sc, 0);
7519         return (rc);
7520 }
7521
7522 static int
7523 load_fw(struct adapter *sc, struct t4_data *fw)
7524 {
7525         int rc;
7526         uint8_t *fw_data;
7527
7528         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7529         if (rc)
7530                 return (rc);
7531
7532         if (sc->flags & FULL_INIT_DONE) {
7533                 rc = EBUSY;
7534                 goto done;
7535         }
7536
7537         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7538         if (fw_data == NULL) {
7539                 rc = ENOMEM;
7540                 goto done;
7541         }
7542
7543         rc = copyin(fw->data, fw_data, fw->len);
7544         if (rc == 0)
7545                 rc = -t4_load_fw(sc, fw_data, fw->len);
7546
7547         free(fw_data, M_CXGBE);
7548 done:
7549         end_synchronized_op(sc, 0);
7550         return (rc);
7551 }
7552
7553 static int
7554 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7555 {
7556         uint32_t addr, off, remaining, i, n;
7557         uint32_t *buf, *b;
7558         uint32_t mw_base, mw_aperture;
7559         int rc;
7560         uint8_t *dst;
7561
7562         rc = validate_mem_range(sc, mr->addr, mr->len);
7563         if (rc != 0)
7564                 return (rc);
7565
7566         memwin_info(sc, win, &mw_base, &mw_aperture);
7567         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7568         addr = mr->addr;
7569         remaining = mr->len;
7570         dst = (void *)mr->data;
7571
7572         while (remaining) {
7573                 off = position_memwin(sc, win, addr);
7574
7575                 /* number of bytes that we'll copy in the inner loop */
7576                 n = min(remaining, mw_aperture - off);
7577                 for (i = 0; i < n; i += 4)
7578                         *b++ = t4_read_reg(sc, mw_base + off + i);
7579
7580                 rc = copyout(buf, dst, n);
7581                 if (rc != 0)
7582                         break;
7583
7584                 b = buf;
7585                 dst += n;
7586                 remaining -= n;
7587                 addr += n;
7588         }
7589
7590         free(buf, M_CXGBE);
7591         return (rc);
7592 }
7593
7594 static int
7595 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7596 {
7597         int rc;
7598
7599         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7600                 return (EINVAL);
7601
7602         if (i2cd->len > sizeof(i2cd->data))
7603                 return (EFBIG);
7604
7605         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7606         if (rc)
7607                 return (rc);
7608         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7609             i2cd->offset, i2cd->len, &i2cd->data[0]);
7610         end_synchronized_op(sc, 0);
7611
7612         return (rc);
7613 }
7614
7615 static int
7616 in_range(int val, int lo, int hi)
7617 {
7618
7619         return (val < 0 || (val <= hi && val >= lo));
7620 }
7621
7622 static int
7623 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7624 {
7625         int fw_subcmd, fw_type, rc;
7626
7627         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7628         if (rc)
7629                 return (rc);
7630
7631         if (!(sc->flags & FULL_INIT_DONE)) {
7632                 rc = EAGAIN;
7633                 goto done;
7634         }
7635
7636         /*
7637          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7638          * sub-command and type are in common locations.)
7639          */
7640         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7641                 fw_subcmd = FW_SCHED_SC_CONFIG;
7642         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7643                 fw_subcmd = FW_SCHED_SC_PARAMS;
7644         else {
7645                 rc = EINVAL;
7646                 goto done;
7647         }
7648         if (p->type == SCHED_CLASS_TYPE_PACKET)
7649                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7650         else {
7651                 rc = EINVAL;
7652                 goto done;
7653         }
7654
7655         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7656                 /* Vet our parameters ..*/
7657                 if (p->u.config.minmax < 0) {
7658                         rc = EINVAL;
7659                         goto done;
7660                 }
7661
7662                 /* And pass the request to the firmware ...*/
7663                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7664                 goto done;
7665         }
7666
7667         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7668                 int fw_level;
7669                 int fw_mode;
7670                 int fw_rateunit;
7671                 int fw_ratemode;
7672
7673                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7674                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7675                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7676                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7677                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7678                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7679                 else {
7680                         rc = EINVAL;
7681                         goto done;
7682                 }
7683
7684                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7685                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7686                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7687                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7688                 else {
7689                         rc = EINVAL;
7690                         goto done;
7691                 }
7692
7693                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7694                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7695                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7696                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7697                 else {
7698                         rc = EINVAL;
7699                         goto done;
7700                 }
7701
7702                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7703                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7704                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7705                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7706                 else {
7707                         rc = EINVAL;
7708                         goto done;
7709                 }
7710
7711                 /* Vet our parameters ... */
7712                 if (!in_range(p->u.params.channel, 0, 3) ||
7713                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7714                     !in_range(p->u.params.minrate, 0, 10000000) ||
7715                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7716                     !in_range(p->u.params.weight, 0, 100)) {
7717                         rc = ERANGE;
7718                         goto done;
7719                 }
7720
7721                 /*
7722                  * Translate any unset parameters into the firmware's
7723                  * nomenclature and/or fail the call if the parameters
7724                  * are required ...
7725                  */
7726                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7727                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7728                         rc = EINVAL;
7729                         goto done;
7730                 }
7731                 if (p->u.params.minrate < 0)
7732                         p->u.params.minrate = 0;
7733                 if (p->u.params.maxrate < 0) {
7734                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7735                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7736                                 rc = EINVAL;
7737                                 goto done;
7738                         } else
7739                                 p->u.params.maxrate = 0;
7740                 }
7741                 if (p->u.params.weight < 0) {
7742                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7743                                 rc = EINVAL;
7744                                 goto done;
7745                         } else
7746                                 p->u.params.weight = 0;
7747                 }
7748                 if (p->u.params.pktsize < 0) {
7749                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7750                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7751                                 rc = EINVAL;
7752                                 goto done;
7753                         } else
7754                                 p->u.params.pktsize = 0;
7755                 }
7756
7757                 /* See what the firmware thinks of the request ... */
7758                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7759                     fw_rateunit, fw_ratemode, p->u.params.channel,
7760                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7761                     p->u.params.weight, p->u.params.pktsize, 1);
7762                 goto done;
7763         }
7764
7765         rc = EINVAL;
7766 done:
7767         end_synchronized_op(sc, 0);
7768         return (rc);
7769 }
7770
7771 static int
7772 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7773 {
7774         struct port_info *pi = NULL;
7775         struct sge_txq *txq;
7776         uint32_t fw_mnem, fw_queue, fw_class;
7777         int i, rc;
7778
7779         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7780         if (rc)
7781                 return (rc);
7782
7783         if (!(sc->flags & FULL_INIT_DONE)) {
7784                 rc = EAGAIN;
7785                 goto done;
7786         }
7787
7788         if (p->port >= sc->params.nports) {
7789                 rc = EINVAL;
7790                 goto done;
7791         }
7792
7793         pi = sc->port[p->port];
7794         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7795                 rc = EINVAL;
7796                 goto done;
7797         }
7798
7799         /*
7800          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7801          * Scheduling Class in this case).
7802          */
7803         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7804             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7805         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7806
7807         /*
7808          * If op.queue is non-negative, then we're only changing the scheduling
7809          * on a single specified TX queue.
7810          */
7811         if (p->queue >= 0) {
7812                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7813                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7814                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7815                     &fw_class);
7816                 goto done;
7817         }
7818
7819         /*
7820          * Change the scheduling on all the TX queues for the
7821          * interface.
7822          */
7823         for_each_txq(pi, i, txq) {
7824                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7825                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7826                     &fw_class);
7827                 if (rc)
7828                         goto done;
7829         }
7830
7831         rc = 0;
7832 done:
7833         end_synchronized_op(sc, 0);
7834         return (rc);
7835 }
7836
7837 int
7838 t4_os_find_pci_capability(struct adapter *sc, int cap)
7839 {
7840         int i;
7841
7842         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7843 }
7844
7845 int
7846 t4_os_pci_save_state(struct adapter *sc)
7847 {
7848         device_t dev;
7849         struct pci_devinfo *dinfo;
7850
7851         dev = sc->dev;
7852         dinfo = device_get_ivars(dev);
7853
7854         pci_cfg_save(dev, dinfo, 0);
7855         return (0);
7856 }
7857
7858 int
7859 t4_os_pci_restore_state(struct adapter *sc)
7860 {
7861         device_t dev;
7862         struct pci_devinfo *dinfo;
7863
7864         dev = sc->dev;
7865         dinfo = device_get_ivars(dev);
7866
7867         pci_cfg_restore(dev, dinfo);
7868         return (0);
7869 }
7870
7871 void
7872 t4_os_portmod_changed(const struct adapter *sc, int idx)
7873 {
7874         struct port_info *pi = sc->port[idx];
7875         static const char *mod_str[] = {
7876                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7877         };
7878
7879         build_medialist(pi, &pi->media);
7880 #ifdef DEV_NETMAP
7881         build_medialist(pi, &pi->nm_media);
7882 #endif
7883
7884         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7885                 if_printf(pi->ifp, "transceiver unplugged.\n");
7886         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7887                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7888         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7889                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7890         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7891                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7892                     mod_str[pi->mod_type]);
7893         } else {
7894                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7895                     pi->mod_type);
7896         }
7897 }
7898
7899 void
7900 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7901 {
7902         struct port_info *pi = sc->port[idx];
7903         struct ifnet *ifp = pi->ifp;
7904
7905         if (link_stat) {
7906                 pi->linkdnrc = -1;
7907                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7908                 if_link_state_change(ifp, LINK_STATE_UP);
7909         } else {
7910                 if (reason >= 0)
7911                         pi->linkdnrc = reason;
7912                 if_link_state_change(ifp, LINK_STATE_DOWN);
7913         }
7914 }
7915
7916 void
7917 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7918 {
7919         struct adapter *sc;
7920
7921         sx_slock(&t4_list_lock);
7922         SLIST_FOREACH(sc, &t4_list, link) {
7923                 /*
7924                  * func should not make any assumptions about what state sc is
7925                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7926                  */
7927                 func(sc, arg);
7928         }
7929         sx_sunlock(&t4_list_lock);
7930 }
7931
7932 static int
7933 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7934 {
7935        return (0);
7936 }
7937
7938 static int
7939 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7940 {
7941        return (0);
7942 }
7943
7944 static int
7945 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7946     struct thread *td)
7947 {
7948         int rc;
7949         struct adapter *sc = dev->si_drv1;
7950
7951         rc = priv_check(td, PRIV_DRIVER);
7952         if (rc != 0)
7953                 return (rc);
7954
7955         switch (cmd) {
7956         case CHELSIO_T4_GETREG: {
7957                 struct t4_reg *edata = (struct t4_reg *)data;
7958
7959                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7960                         return (EFAULT);
7961
7962                 if (edata->size == 4)
7963                         edata->val = t4_read_reg(sc, edata->addr);
7964                 else if (edata->size == 8)
7965                         edata->val = t4_read_reg64(sc, edata->addr);
7966                 else
7967                         return (EINVAL);
7968
7969                 break;
7970         }
7971         case CHELSIO_T4_SETREG: {
7972                 struct t4_reg *edata = (struct t4_reg *)data;
7973
7974                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7975                         return (EFAULT);
7976
7977                 if (edata->size == 4) {
7978                         if (edata->val & 0xffffffff00000000)
7979                                 return (EINVAL);
7980                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7981                 } else if (edata->size == 8)
7982                         t4_write_reg64(sc, edata->addr, edata->val);
7983                 else
7984                         return (EINVAL);
7985                 break;
7986         }
7987         case CHELSIO_T4_REGDUMP: {
7988                 struct t4_regdump *regs = (struct t4_regdump *)data;
7989                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7990                 uint8_t *buf;
7991
7992                 if (regs->len < reglen) {
7993                         regs->len = reglen; /* hint to the caller */
7994                         return (ENOBUFS);
7995                 }
7996
7997                 regs->len = reglen;
7998                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7999                 t4_get_regs(sc, regs, buf);
8000                 rc = copyout(buf, regs->data, reglen);
8001                 free(buf, M_CXGBE);
8002                 break;
8003         }
8004         case CHELSIO_T4_GET_FILTER_MODE:
8005                 rc = get_filter_mode(sc, (uint32_t *)data);
8006                 break;
8007         case CHELSIO_T4_SET_FILTER_MODE:
8008                 rc = set_filter_mode(sc, *(uint32_t *)data);
8009                 break;
8010         case CHELSIO_T4_GET_FILTER:
8011                 rc = get_filter(sc, (struct t4_filter *)data);
8012                 break;
8013         case CHELSIO_T4_SET_FILTER:
8014                 rc = set_filter(sc, (struct t4_filter *)data);
8015                 break;
8016         case CHELSIO_T4_DEL_FILTER:
8017                 rc = del_filter(sc, (struct t4_filter *)data);
8018                 break;
8019         case CHELSIO_T4_GET_SGE_CONTEXT:
8020                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8021                 break;
8022         case CHELSIO_T4_LOAD_FW:
8023                 rc = load_fw(sc, (struct t4_data *)data);
8024                 break;
8025         case CHELSIO_T4_GET_MEM:
8026                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8027                 break;
8028         case CHELSIO_T4_GET_I2C:
8029                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8030                 break;
8031         case CHELSIO_T4_CLEAR_STATS: {
8032                 int i;
8033                 u_int port_id = *(uint32_t *)data;
8034                 struct port_info *pi;
8035
8036                 if (port_id >= sc->params.nports)
8037                         return (EINVAL);
8038                 pi = sc->port[port_id];
8039
8040                 /* MAC stats */
8041                 t4_clr_port_stats(sc, pi->tx_chan);
8042                 pi->tx_parse_error = 0;
8043
8044                 if (pi->flags & PORT_INIT_DONE) {
8045                         struct sge_rxq *rxq;
8046                         struct sge_txq *txq;
8047                         struct sge_wrq *wrq;
8048
8049                         for_each_rxq(pi, i, rxq) {
8050 #if defined(INET) || defined(INET6)
8051                                 rxq->lro.lro_queued = 0;
8052                                 rxq->lro.lro_flushed = 0;
8053 #endif
8054                                 rxq->rxcsum = 0;
8055                                 rxq->vlan_extraction = 0;
8056                         }
8057
8058                         for_each_txq(pi, i, txq) {
8059                                 txq->txcsum = 0;
8060                                 txq->tso_wrs = 0;
8061                                 txq->vlan_insertion = 0;
8062                                 txq->imm_wrs = 0;
8063                                 txq->sgl_wrs = 0;
8064                                 txq->txpkt_wrs = 0;
8065                                 txq->txpkts0_wrs = 0;
8066                                 txq->txpkts1_wrs = 0;
8067                                 txq->txpkts0_pkts = 0;
8068                                 txq->txpkts1_pkts = 0;
8069                                 mp_ring_reset_stats(txq->r);
8070                         }
8071
8072 #ifdef TCP_OFFLOAD
8073                         /* nothing to clear for each ofld_rxq */
8074
8075                         for_each_ofld_txq(pi, i, wrq) {
8076                                 wrq->tx_wrs_direct = 0;
8077                                 wrq->tx_wrs_copied = 0;
8078                         }
8079 #endif
8080                         wrq = &sc->sge.ctrlq[pi->port_id];
8081                         wrq->tx_wrs_direct = 0;
8082                         wrq->tx_wrs_copied = 0;
8083                 }
8084                 break;
8085         }
8086         case CHELSIO_T4_SCHED_CLASS:
8087                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8088                 break;
8089         case CHELSIO_T4_SCHED_QUEUE:
8090                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8091                 break;
8092         case CHELSIO_T4_GET_TRACER:
8093                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8094                 break;
8095         case CHELSIO_T4_SET_TRACER:
8096                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8097                 break;
8098         default:
8099                 rc = EINVAL;
8100         }
8101
8102         return (rc);
8103 }
8104
8105 #ifdef TCP_OFFLOAD
8106 void
8107 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8108     const unsigned int *pgsz_order)
8109 {
8110         struct port_info *pi = ifp->if_softc;
8111         struct adapter *sc = pi->adapter;
8112
8113         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8114         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8115                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8116                 V_HPZ3(pgsz_order[3]));
8117 }
8118
8119 static int
8120 toe_capability(struct port_info *pi, int enable)
8121 {
8122         int rc;
8123         struct adapter *sc = pi->adapter;
8124
8125         ASSERT_SYNCHRONIZED_OP(sc);
8126
8127         if (!is_offload(sc))
8128                 return (ENODEV);
8129
8130         if (enable) {
8131                 /*
8132                  * We need the port's queues around so that we're able to send
8133                  * and receive CPLs to/from the TOE even if the ifnet for this
8134                  * port has never been UP'd administratively.
8135                  */
8136                 if (!(pi->flags & PORT_INIT_DONE)) {
8137                         rc = cxgbe_init_synchronized(pi);
8138                         if (rc)
8139                                 return (rc);
8140                 }
8141
8142                 if (isset(&sc->offload_map, pi->port_id))
8143                         return (0);
8144
8145                 if (!uld_active(sc, ULD_TOM)) {
8146                         rc = t4_activate_uld(sc, ULD_TOM);
8147                         if (rc == EAGAIN) {
8148                                 log(LOG_WARNING,
8149                                     "You must kldload t4_tom.ko before trying "
8150                                     "to enable TOE on a cxgbe interface.\n");
8151                         }
8152                         if (rc != 0)
8153                                 return (rc);
8154                         KASSERT(sc->tom_softc != NULL,
8155                             ("%s: TOM activated but softc NULL", __func__));
8156                         KASSERT(uld_active(sc, ULD_TOM),
8157                             ("%s: TOM activated but flag not set", __func__));
8158                 }
8159
8160                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8161                 if (!uld_active(sc, ULD_IWARP))
8162                         (void) t4_activate_uld(sc, ULD_IWARP);
8163                 if (!uld_active(sc, ULD_ISCSI))
8164                         (void) t4_activate_uld(sc, ULD_ISCSI);
8165
8166                 setbit(&sc->offload_map, pi->port_id);
8167         } else {
8168                 if (!isset(&sc->offload_map, pi->port_id))
8169                         return (0);
8170
8171                 KASSERT(uld_active(sc, ULD_TOM),
8172                     ("%s: TOM never initialized?", __func__));
8173                 clrbit(&sc->offload_map, pi->port_id);
8174         }
8175
8176         return (0);
8177 }
8178
8179 /*
8180  * Add an upper layer driver to the global list.
8181  */
8182 int
8183 t4_register_uld(struct uld_info *ui)
8184 {
8185         int rc = 0;
8186         struct uld_info *u;
8187
8188         sx_xlock(&t4_uld_list_lock);
8189         SLIST_FOREACH(u, &t4_uld_list, link) {
8190             if (u->uld_id == ui->uld_id) {
8191                     rc = EEXIST;
8192                     goto done;
8193             }
8194         }
8195
8196         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8197         ui->refcount = 0;
8198 done:
8199         sx_xunlock(&t4_uld_list_lock);
8200         return (rc);
8201 }
8202
8203 int
8204 t4_unregister_uld(struct uld_info *ui)
8205 {
8206         int rc = EINVAL;
8207         struct uld_info *u;
8208
8209         sx_xlock(&t4_uld_list_lock);
8210
8211         SLIST_FOREACH(u, &t4_uld_list, link) {
8212             if (u == ui) {
8213                     if (ui->refcount > 0) {
8214                             rc = EBUSY;
8215                             goto done;
8216                     }
8217
8218                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8219                     rc = 0;
8220                     goto done;
8221             }
8222         }
8223 done:
8224         sx_xunlock(&t4_uld_list_lock);
8225         return (rc);
8226 }
8227
8228 int
8229 t4_activate_uld(struct adapter *sc, int id)
8230 {
8231         int rc;
8232         struct uld_info *ui;
8233
8234         ASSERT_SYNCHRONIZED_OP(sc);
8235
8236         if (id < 0 || id > ULD_MAX)
8237                 return (EINVAL);
8238         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8239
8240         sx_slock(&t4_uld_list_lock);
8241
8242         SLIST_FOREACH(ui, &t4_uld_list, link) {
8243                 if (ui->uld_id == id) {
8244                         if (!(sc->flags & FULL_INIT_DONE)) {
8245                                 rc = adapter_full_init(sc);
8246                                 if (rc != 0)
8247                                         break;
8248                         }
8249
8250                         rc = ui->activate(sc);
8251                         if (rc == 0) {
8252                                 setbit(&sc->active_ulds, id);
8253                                 ui->refcount++;
8254                         }
8255                         break;
8256                 }
8257         }
8258
8259         sx_sunlock(&t4_uld_list_lock);
8260
8261         return (rc);
8262 }
8263
8264 int
8265 t4_deactivate_uld(struct adapter *sc, int id)
8266 {
8267         int rc;
8268         struct uld_info *ui;
8269
8270         ASSERT_SYNCHRONIZED_OP(sc);
8271
8272         if (id < 0 || id > ULD_MAX)
8273                 return (EINVAL);
8274         rc = ENXIO;
8275
8276         sx_slock(&t4_uld_list_lock);
8277
8278         SLIST_FOREACH(ui, &t4_uld_list, link) {
8279                 if (ui->uld_id == id) {
8280                         rc = ui->deactivate(sc);
8281                         if (rc == 0) {
8282                                 clrbit(&sc->active_ulds, id);
8283                                 ui->refcount--;
8284                         }
8285                         break;
8286                 }
8287         }
8288
8289         sx_sunlock(&t4_uld_list_lock);
8290
8291         return (rc);
8292 }
8293
8294 int
8295 uld_active(struct adapter *sc, int uld_id)
8296 {
8297
8298         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8299
8300         return (isset(&sc->active_ulds, uld_id));
8301 }
8302 #endif
8303
8304 /*
8305  * Come up with reasonable defaults for some of the tunables, provided they're
8306  * not set by the user (in which case we'll use the values as is).
8307  */
8308 static void
8309 tweak_tunables(void)
8310 {
8311         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8312
8313         if (t4_ntxq10g < 1)
8314                 t4_ntxq10g = min(nc, NTXQ_10G);
8315
8316         if (t4_ntxq1g < 1)
8317                 t4_ntxq1g = min(nc, NTXQ_1G);
8318
8319         if (t4_nrxq10g < 1)
8320                 t4_nrxq10g = min(nc, NRXQ_10G);
8321
8322         if (t4_nrxq1g < 1)
8323                 t4_nrxq1g = min(nc, NRXQ_1G);
8324
8325 #ifdef TCP_OFFLOAD
8326         if (t4_nofldtxq10g < 1)
8327                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8328
8329         if (t4_nofldtxq1g < 1)
8330                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8331
8332         if (t4_nofldrxq10g < 1)
8333                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8334
8335         if (t4_nofldrxq1g < 1)
8336                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8337
8338         if (t4_toecaps_allowed == -1)
8339                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8340 #else
8341         if (t4_toecaps_allowed == -1)
8342                 t4_toecaps_allowed = 0;
8343 #endif
8344
8345 #ifdef DEV_NETMAP
8346         if (t4_nnmtxq10g < 1)
8347                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8348
8349         if (t4_nnmtxq1g < 1)
8350                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8351
8352         if (t4_nnmrxq10g < 1)
8353                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8354
8355         if (t4_nnmrxq1g < 1)
8356                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8357 #endif
8358
8359         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8360                 t4_tmr_idx_10g = TMR_IDX_10G;
8361
8362         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8363                 t4_pktc_idx_10g = PKTC_IDX_10G;
8364
8365         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8366                 t4_tmr_idx_1g = TMR_IDX_1G;
8367
8368         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8369                 t4_pktc_idx_1g = PKTC_IDX_1G;
8370
8371         if (t4_qsize_txq < 128)
8372                 t4_qsize_txq = 128;
8373
8374         if (t4_qsize_rxq < 128)
8375                 t4_qsize_rxq = 128;
8376         while (t4_qsize_rxq & 7)
8377                 t4_qsize_rxq++;
8378
8379         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8380 }
8381
8382 static struct sx mlu;   /* mod load unload */
8383 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8384
8385 static int
8386 mod_event(module_t mod, int cmd, void *arg)
8387 {
8388         int rc = 0;
8389         static int loaded = 0;
8390
8391         switch (cmd) {
8392         case MOD_LOAD:
8393                 sx_xlock(&mlu);
8394                 if (loaded++ == 0) {
8395                         t4_sge_modload();
8396                         sx_init(&t4_list_lock, "T4/T5 adapters");
8397                         SLIST_INIT(&t4_list);
8398 #ifdef TCP_OFFLOAD
8399                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8400                         SLIST_INIT(&t4_uld_list);
8401 #endif
8402                         t4_tracer_modload();
8403                         tweak_tunables();
8404                 }
8405                 sx_xunlock(&mlu);
8406                 break;
8407
8408         case MOD_UNLOAD:
8409                 sx_xlock(&mlu);
8410                 if (--loaded == 0) {
8411                         int tries;
8412
8413                         sx_slock(&t4_list_lock);
8414                         if (!SLIST_EMPTY(&t4_list)) {
8415                                 rc = EBUSY;
8416                                 sx_sunlock(&t4_list_lock);
8417                                 goto done_unload;
8418                         }
8419 #ifdef TCP_OFFLOAD
8420                         sx_slock(&t4_uld_list_lock);
8421                         if (!SLIST_EMPTY(&t4_uld_list)) {
8422                                 rc = EBUSY;
8423                                 sx_sunlock(&t4_uld_list_lock);
8424                                 sx_sunlock(&t4_list_lock);
8425                                 goto done_unload;
8426                         }
8427 #endif
8428                         tries = 0;
8429                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8430                                 uprintf("%ju clusters with custom free routine "
8431                                     "still is use.\n", t4_sge_extfree_refs());
8432                                 pause("t4unload", 2 * hz);
8433                         }
8434 #ifdef TCP_OFFLOAD
8435                         sx_sunlock(&t4_uld_list_lock);
8436 #endif
8437                         sx_sunlock(&t4_list_lock);
8438
8439                         if (t4_sge_extfree_refs() == 0) {
8440                                 t4_tracer_modunload();
8441 #ifdef TCP_OFFLOAD
8442                                 sx_destroy(&t4_uld_list_lock);
8443 #endif
8444                                 sx_destroy(&t4_list_lock);
8445                                 t4_sge_modunload();
8446                                 loaded = 0;
8447                         } else {
8448                                 rc = EBUSY;
8449                                 loaded++;       /* undo earlier decrement */
8450                         }
8451                 }
8452 done_unload:
8453                 sx_xunlock(&mlu);
8454                 break;
8455         }
8456
8457         return (rc);
8458 }
8459
8460 static devclass_t t4_devclass, t5_devclass;
8461 static devclass_t cxgbe_devclass, cxl_devclass;
8462
8463 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8464 MODULE_VERSION(t4nex, 1);
8465 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8466
8467 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8468 MODULE_VERSION(t5nex, 1);
8469 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8470
8471 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8472 MODULE_VERSION(cxgbe, 1);
8473
8474 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8475 MODULE_VERSION(cxl, 1);