sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static int
 574 t4_attach(device_t dev)
 575 {
 576         struct adapter *sc;
 577         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 578         struct intrs_and_queues iaq;
 579         struct sge *s;
 580 #ifdef TCP_OFFLOAD
 581         int ofld_rqidx, ofld_tqidx;
 582 #endif
 583 #ifdef DEV_NETMAP
 584         int nm_rqidx, nm_tqidx;
 585 #endif
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 590
 591         pci_enable_busmaster(dev);
 592         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 593                 uint32_t v;
 594
 595                 pci_set_max_read_req(dev, 4096);
 596                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 597                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 598                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 599
 600                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 601         }
 602
 603         sc->traceq = -1;
 604         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 605         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 606             device_get_nameunit(dev));
 607
 608         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 609             device_get_nameunit(dev));
 610         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 611         sx_xlock(&t4_list_lock);
 612         SLIST_INSERT_HEAD(&t4_list, sc, link);
 613         sx_xunlock(&t4_list_lock);
 614
 615         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 616         TAILQ_INIT(&sc->sfl);
 617         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 618
 619         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 620
 621         rc = map_bars_0_and_4(sc);
 622         if (rc != 0)
 623                 goto done; /* error message displayed already */
 624
 625         /*
 626          * This is the real PF# to which we're attaching.  Works from within PCI
 627          * passthrough environments too, where pci_get_function() could return a
 628          * different PF# depending on the passthrough configuration.  We need to
 629          * use the real PF# in all our communication with the firmware.
 630          */
 631         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 632         sc->mbox = sc->pf;
 633
 634         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 635         sc->an_handler = an_not_handled;
 636         for (i = 0; i < nitems(sc->cpl_handler); i++)
 637                 sc->cpl_handler[i] = cpl_not_handled;
 638         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 639                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 640         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 642         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 643         t4_init_sge_cpl_handlers(sc);
 644
 645         /* Prepare the adapter for operation */
 646         rc = -t4_prep_adapter(sc);
 647         if (rc != 0) {
 648                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 649                 goto done;
 650         }
 651
 652         /*
 653          * Do this really early, with the memory windows set up even before the
 654          * character device.  The userland tool's register i/o and mem read
 655          * will work even in "recovery mode".
 656          */
 657         setup_memwin(sc);
 658         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 659             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 660             device_get_nameunit(dev));
 661         if (sc->cdev == NULL)
 662                 device_printf(dev, "failed to create nexus char device.\n");
 663         else
 664                 sc->cdev->si_drv1 = sc;
 665
 666         /* Go no further if recovery mode has been requested. */
 667         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 668                 device_printf(dev, "recovery mode.\n");
 669                 goto done;
 670         }
 671
 672 #if defined(__i386__)
 673         if ((cpu_feature & CPUID_CX8) == 0) {
 674                 device_printf(dev, "64 bit atomics not available.\n");
 675                 rc = ENOTSUP;
 676                 goto done;
 677         }
 678 #endif
 679
 680         /* Prepare the firmware for operation */
 681         rc = prep_firmware(sc);
 682         if (rc != 0)
 683                 goto done; /* error message displayed already */
 684
 685         rc = get_params__post_init(sc);
 686         if (rc != 0)
 687                 goto done; /* error message displayed already */
 688
 689         rc = set_params__post_init(sc);
 690         if (rc != 0)
 691                 goto done; /* error message displayed already */
 692
 693         rc = map_bar_2(sc);
 694         if (rc != 0)
 695                 goto done; /* error message displayed already */
 696
 697         rc = t4_create_dma_tag(sc);
 698         if (rc != 0)
 699                 goto done; /* error message displayed already */
 700
 701         /*
 702          * First pass over all the ports - allocate VIs and initialize some
 703          * basic parameters like mac address, port type, etc.  We also figure
 704          * out whether a port is 10G or 1G and use that information when
 705          * calculating how many interrupts to attempt to allocate.
 706          */
 707         n10g = n1g = 0;
 708         for_each_port(sc, i) {
 709                 struct port_info *pi;
 710
 711                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 712                 sc->port[i] = pi;
 713
 714                 /* These must be set before t4_port_init */
 715                 pi->adapter = sc;
 716                 pi->port_id = i;
 717
 718                 /* Allocate the vi and initialize parameters like mac addr */
 719                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 720                 if (rc != 0) {
 721                         device_printf(dev, "unable to initialize port %d: %d\n",
 722                             i, rc);
 723                         free(pi, M_CXGBE);
 724                         sc->port[i] = NULL;
 725                         goto done;
 726                 }
 727
 728                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 729                 pi->link_cfg.requested_fc |= t4_pause_settings;
 730                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 731                 pi->link_cfg.fc |= t4_pause_settings;
 732
 733                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 734                 if (rc != 0) {
 735                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 736                         free(pi, M_CXGBE);
 737                         sc->port[i] = NULL;
 738                         goto done;
 739                 }
 740
 741                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 742                     device_get_nameunit(dev), i);
 743                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 744                 sc->chan_map[pi->tx_chan] = i;
 745
 746                 if (is_10G_port(pi) || is_40G_port(pi)) {
 747                         n10g++;
 748                         pi->tmr_idx = t4_tmr_idx_10g;
 749                         pi->pktc_idx = t4_pktc_idx_10g;
 750                 } else {
 751                         n1g++;
 752                         pi->tmr_idx = t4_tmr_idx_1g;
 753                         pi->pktc_idx = t4_pktc_idx_1g;
 754                 }
 755
 756                 pi->xact_addr_filt = -1;
 757                 pi->linkdnrc = -1;
 758
 759                 pi->qsize_rxq = t4_qsize_rxq;
 760                 pi->qsize_txq = t4_qsize_txq;
 761
 762                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 763                 if (pi->dev == NULL) {
 764                         device_printf(dev,
 765                             "failed to add device for port %d.\n", i);
 766                         rc = ENXIO;
 767                         goto done;
 768                 }
 769                 device_set_softc(pi->dev, pi);
 770         }
 771
 772         /*
 773          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 774          */
 775         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 776         if (rc != 0)
 777                 goto done; /* error message displayed already */
 778
 779         sc->intr_type = iaq.intr_type;
 780         sc->intr_count = iaq.nirq;
 781
 782         s = &sc->sge;
 783         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 784         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 785         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 786         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 787         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 788 #ifdef TCP_OFFLOAD
 789         if (is_offload(sc)) {
 790                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 791                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 792                 s->neq += s->nofldtxq + s->nofldrxq;
 793                 s->niq += s->nofldrxq;
 794
 795                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 796                     M_CXGBE, M_ZERO | M_WAITOK);
 797                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 798                     M_CXGBE, M_ZERO | M_WAITOK);
 799         }
 800 #endif
 801 #ifdef DEV_NETMAP
 802         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 803         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 804         s->neq += s->nnmtxq + s->nnmrxq;
 805         s->niq += s->nnmrxq;
 806
 807         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 808             M_CXGBE, M_ZERO | M_WAITOK);
 809         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 810             M_CXGBE, M_ZERO | M_WAITOK);
 811 #endif
 812
 813         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 814             M_ZERO | M_WAITOK);
 815         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 816             M_ZERO | M_WAITOK);
 817         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 818             M_ZERO | M_WAITOK);
 819         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 820             M_ZERO | M_WAITOK);
 821         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 822             M_ZERO | M_WAITOK);
 823
 824         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 825             M_ZERO | M_WAITOK);
 826
 827         t4_init_l2t(sc, M_WAITOK);
 828
 829         /*
 830          * Second pass over the ports.  This time we know the number of rx and
 831          * tx queues that each port should get.
 832          */
 833         rqidx = tqidx = 0;
 834 #ifdef TCP_OFFLOAD
 835         ofld_rqidx = ofld_tqidx = 0;
 836 #endif
 837 #ifdef DEV_NETMAP
 838         nm_rqidx = nm_tqidx = 0;
 839 #endif
 840         for_each_port(sc, i) {
 841                 struct port_info *pi = sc->port[i];
 842
 843                 if (pi == NULL)
 844                         continue;
 845
 846                 pi->first_rxq = rqidx;
 847                 pi->first_txq = tqidx;
 848                 if (is_10G_port(pi) || is_40G_port(pi)) {
 849                         pi->flags |= iaq.intr_flags_10g;
 850                         pi->nrxq = iaq.nrxq10g;
 851                         pi->ntxq = iaq.ntxq10g;
 852                 } else {
 853                         pi->flags |= iaq.intr_flags_1g;
 854                         pi->nrxq = iaq.nrxq1g;
 855                         pi->ntxq = iaq.ntxq1g;
 856                 }
 857
 858                 if (pi->ntxq > 1)
 859                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 860                 else
 861                         pi->rsrv_noflowq = 0;
 862
 863                 rqidx += pi->nrxq;
 864                 tqidx += pi->ntxq;
 865 #ifdef TCP_OFFLOAD
 866                 if (is_offload(sc)) {
 867                         pi->first_ofld_rxq = ofld_rqidx;
 868                         pi->first_ofld_txq = ofld_tqidx;
 869                         if (is_10G_port(pi) || is_40G_port(pi)) {
 870                                 pi->nofldrxq = iaq.nofldrxq10g;
 871                                 pi->nofldtxq = iaq.nofldtxq10g;
 872                         } else {
 873                                 pi->nofldrxq = iaq.nofldrxq1g;
 874                                 pi->nofldtxq = iaq.nofldtxq1g;
 875                         }
 876                         ofld_rqidx += pi->nofldrxq;
 877                         ofld_tqidx += pi->nofldtxq;
 878                 }
 879 #endif
 880 #ifdef DEV_NETMAP
 881                 pi->first_nm_rxq = nm_rqidx;
 882                 pi->first_nm_txq = nm_tqidx;
 883                 if (is_10G_port(pi) || is_40G_port(pi)) {
 884                         pi->nnmrxq = iaq.nnmrxq10g;
 885                         pi->nnmtxq = iaq.nnmtxq10g;
 886                 } else {
 887                         pi->nnmrxq = iaq.nnmrxq1g;
 888                         pi->nnmtxq = iaq.nnmtxq1g;
 889                 }
 890                 nm_rqidx += pi->nnmrxq;
 891                 nm_tqidx += pi->nnmtxq;
 892 #endif
 893         }
 894
 895         rc = setup_intr_handlers(sc);
 896         if (rc != 0) {
 897                 device_printf(dev,
 898                     "failed to setup interrupt handlers: %d\n", rc);
 899                 goto done;
 900         }
 901
 902         rc = bus_generic_attach(dev);
 903         if (rc != 0) {
 904                 device_printf(dev,
 905                     "failed to attach all child ports: %d\n", rc);
 906                 goto done;
 907         }
 908
 909         device_printf(dev,
 910             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 911             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 912             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 913             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 914             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 915
 916         t4_set_desc(sc);
 917
 918 done:
 919         if (rc != 0 && sc->cdev) {
 920                 /* cdev was created and so cxgbetool works; recover that way. */
 921                 device_printf(dev,
 922                     "error during attach, adapter is now in recovery mode.\n");
 923                 rc = 0;
 924         }
 925
 926         if (rc != 0)
 927                 t4_detach(dev);
 928         else
 929                 t4_sysctls(sc);
 930
 931         return (rc);
 932 }
 933
 934 /*
 935  * Idempotent
 936  */
 937 static int
 938 t4_detach(device_t dev)
 939 {
 940         struct adapter *sc;
 941         struct port_info *pi;
 942         int i, rc;
 943
 944         sc = device_get_softc(dev);
 945
 946         if (sc->flags & FULL_INIT_DONE)
 947                 t4_intr_disable(sc);
 948
 949         if (sc->cdev) {
 950                 destroy_dev(sc->cdev);
 951                 sc->cdev = NULL;
 952         }
 953
 954         rc = bus_generic_detach(dev);
 955         if (rc) {
 956                 device_printf(dev,
 957                     "failed to detach child devices: %d\n", rc);
 958                 return (rc);
 959         }
 960
 961         for (i = 0; i < sc->intr_count; i++)
 962                 t4_free_irq(sc, &sc->irq[i]);
 963
 964         for (i = 0; i < MAX_NPORTS; i++) {
 965                 pi = sc->port[i];
 966                 if (pi) {
 967                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 968                         if (pi->dev)
 969                                 device_delete_child(dev, pi->dev);
 970
 971                         mtx_destroy(&pi->pi_lock);
 972                         free(pi, M_CXGBE);
 973                 }
 974         }
 975
 976         if (sc->flags & FULL_INIT_DONE)
 977                 adapter_full_uninit(sc);
 978
 979         if (sc->flags & FW_OK)
 980                 t4_fw_bye(sc, sc->mbox);
 981
 982         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 983                 pci_release_msi(dev);
 984
 985         if (sc->regs_res)
 986                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 987                     sc->regs_res);
 988
 989         if (sc->udbs_res)
 990                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 991                     sc->udbs_res);
 992
 993         if (sc->msix_res)
 994                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 995                     sc->msix_res);
 996
 997         if (sc->l2t)
 998                 t4_free_l2t(sc->l2t);
 999
1000 #ifdef TCP_OFFLOAD
1001         free(sc->sge.ofld_rxq, M_CXGBE);
1002         free(sc->sge.ofld_txq, M_CXGBE);
1003 #endif
1004 #ifdef DEV_NETMAP
1005         free(sc->sge.nm_rxq, M_CXGBE);
1006         free(sc->sge.nm_txq, M_CXGBE);
1007 #endif
1008         free(sc->irq, M_CXGBE);
1009         free(sc->sge.rxq, M_CXGBE);
1010         free(sc->sge.txq, M_CXGBE);
1011         free(sc->sge.ctrlq, M_CXGBE);
1012         free(sc->sge.iqmap, M_CXGBE);
1013         free(sc->sge.eqmap, M_CXGBE);
1014         free(sc->tids.ftid_tab, M_CXGBE);
1015         t4_destroy_dma_tag(sc);
1016         if (mtx_initialized(&sc->sc_lock)) {
1017                 sx_xlock(&t4_list_lock);
1018                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1019                 sx_xunlock(&t4_list_lock);
1020                 mtx_destroy(&sc->sc_lock);
1021         }
1022
1023         if (mtx_initialized(&sc->tids.ftid_lock))
1024                 mtx_destroy(&sc->tids.ftid_lock);
1025         if (mtx_initialized(&sc->sfl_lock))
1026                 mtx_destroy(&sc->sfl_lock);
1027         if (mtx_initialized(&sc->ifp_lock))
1028                 mtx_destroy(&sc->ifp_lock);
1029         if (mtx_initialized(&sc->regwin_lock))
1030                 mtx_destroy(&sc->regwin_lock);
1031
1032         bzero(sc, sizeof(*sc));
1033
1034         return (0);
1035 }
1036
1037 static int
1038 cxgbe_probe(device_t dev)
1039 {
1040         char buf[128];
1041         struct port_info *pi = device_get_softc(dev);
1042
1043         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1044         device_set_desc_copy(dev, buf);
1045
1046         return (BUS_PROBE_DEFAULT);
1047 }
1048
1049 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1050     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1051     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1052 #define T4_CAP_ENABLE (T4_CAP)
1053
1054 static int
1055 cxgbe_attach(device_t dev)
1056 {
1057         struct port_info *pi = device_get_softc(dev);
1058         struct ifnet *ifp;
1059         char *s;
1060         int n, o;
1061
1062         /* Allocate an ifnet and set it up */
1063         ifp = if_alloc(IFT_ETHER);
1064         if (ifp == NULL) {
1065                 device_printf(dev, "Cannot allocate ifnet\n");
1066                 return (ENOMEM);
1067         }
1068         pi->ifp = ifp;
1069         ifp->if_softc = pi;
1070
1071         callout_init(&pi->tick, CALLOUT_MPSAFE);
1072
1073         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1074         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1075
1076         ifp->if_init = cxgbe_init;
1077         ifp->if_ioctl = cxgbe_ioctl;
1078         ifp->if_transmit = cxgbe_transmit;
1079         ifp->if_qflush = cxgbe_qflush;
1080
1081         ifp->if_capabilities = T4_CAP;
1082 #ifdef TCP_OFFLOAD
1083         if (is_offload(pi->adapter))
1084                 ifp->if_capabilities |= IFCAP_TOE;
1085 #endif
1086         ifp->if_capenable = T4_CAP_ENABLE;
1087         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1088             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1089
1090         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1091         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1092         ifp->if_hw_tsomaxsegsize = 65536;
1093
1094         /* Initialize ifmedia for this port */
1095         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1096             cxgbe_media_status);
1097         build_medialist(pi, &pi->media);
1098
1099         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1100             EVENTHANDLER_PRI_ANY);
1101
1102         ether_ifattach(ifp, pi->hw_addr);
1103
1104         n = 128;
1105         s = malloc(n, M_CXGBE, M_WAITOK);
1106         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1107         MPASS(n > o);
1108 #ifdef TCP_OFFLOAD
1109         if (is_offload(pi->adapter)) {
1110                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1111                     pi->nofldtxq, pi->nofldrxq);
1112                 MPASS(n > o);
1113         }
1114 #endif
1115 #ifdef DEV_NETMAP
1116         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1117             pi->nnmrxq);
1118         MPASS(n > o);
1119 #endif
1120         device_printf(dev, "%s\n", s);
1121         free(s, M_CXGBE);
1122
1123 #ifdef DEV_NETMAP
1124         /* nm_media handled here to keep implementation private to this file */
1125         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1126             cxgbe_media_status);
1127         build_medialist(pi, &pi->nm_media);
1128         create_netmap_ifnet(pi);        /* logs errors it something fails */
1129 #endif
1130         cxgbe_sysctls(pi);
1131
1132         return (0);
1133 }
1134
1135 static int
1136 cxgbe_detach(device_t dev)
1137 {
1138         struct port_info *pi = device_get_softc(dev);
1139         struct adapter *sc = pi->adapter;
1140         struct ifnet *ifp = pi->ifp;
1141
1142         /* Tell if_ioctl and if_init that the port is going away */
1143         ADAPTER_LOCK(sc);
1144         SET_DOOMED(pi);
1145         wakeup(&sc->flags);
1146         while (IS_BUSY(sc))
1147                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1148         SET_BUSY(sc);
1149 #ifdef INVARIANTS
1150         sc->last_op = "t4detach";
1151         sc->last_op_thr = curthread;
1152 #endif
1153         ADAPTER_UNLOCK(sc);
1154
1155         if (pi->flags & HAS_TRACEQ) {
1156                 sc->traceq = -1;        /* cloner should not create ifnet */
1157                 t4_tracer_port_detach(sc);
1158         }
1159
1160         if (pi->vlan_c)
1161                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1162
1163         PORT_LOCK(pi);
1164         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1165         callout_stop(&pi->tick);
1166         PORT_UNLOCK(pi);
1167         callout_drain(&pi->tick);
1168
1169         /* Let detach proceed even if these fail. */
1170         cxgbe_uninit_synchronized(pi);
1171         port_full_uninit(pi);
1172
1173         ifmedia_removeall(&pi->media);
1174         ether_ifdetach(pi->ifp);
1175         if_free(pi->ifp);
1176
1177 #ifdef DEV_NETMAP
1178         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1179         destroy_netmap_ifnet(pi);
1180 #endif
1181
1182         ADAPTER_LOCK(sc);
1183         CLR_BUSY(sc);
1184         wakeup(&sc->flags);
1185         ADAPTER_UNLOCK(sc);
1186
1187         return (0);
1188 }
1189
1190 static void
1191 cxgbe_init(void *arg)
1192 {
1193         struct port_info *pi = arg;
1194         struct adapter *sc = pi->adapter;
1195
1196         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1197                 return;
1198         cxgbe_init_synchronized(pi);
1199         end_synchronized_op(sc, 0);
1200 }
1201
1202 static int
1203 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1204 {
1205         int rc = 0, mtu, flags, can_sleep;
1206         struct port_info *pi = ifp->if_softc;
1207         struct adapter *sc = pi->adapter;
1208         struct ifreq *ifr = (struct ifreq *)data;
1209         uint32_t mask;
1210
1211         switch (cmd) {
1212         case SIOCSIFMTU:
1213                 mtu = ifr->ifr_mtu;
1214                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1215                         return (EINVAL);
1216
1217                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1218                 if (rc)
1219                         return (rc);
1220                 ifp->if_mtu = mtu;
1221                 if (pi->flags & PORT_INIT_DONE) {
1222                         t4_update_fl_bufsize(ifp);
1223                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1224                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1225                 }
1226                 end_synchronized_op(sc, 0);
1227                 break;
1228
1229         case SIOCSIFFLAGS:
1230                 can_sleep = 0;
1231 redo_sifflags:
1232                 rc = begin_synchronized_op(sc, pi,
1233                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1234                 if (rc)
1235                         return (rc);
1236
1237                 if (ifp->if_flags & IFF_UP) {
1238                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1239                                 flags = pi->if_flags;
1240                                 if ((ifp->if_flags ^ flags) &
1241                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1242                                         if (can_sleep == 1) {
1243                                                 end_synchronized_op(sc, 0);
1244                                                 can_sleep = 0;
1245                                                 goto redo_sifflags;
1246                                         }
1247                                         rc = update_mac_settings(ifp,
1248                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1249                                 }
1250                         } else {
1251                                 if (can_sleep == 0) {
1252                                         end_synchronized_op(sc, LOCK_HELD);
1253                                         can_sleep = 1;
1254                                         goto redo_sifflags;
1255                                 }
1256                                 rc = cxgbe_init_synchronized(pi);
1257                         }
1258                         pi->if_flags = ifp->if_flags;
1259                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1260                         if (can_sleep == 0) {
1261                                 end_synchronized_op(sc, LOCK_HELD);
1262                                 can_sleep = 1;
1263                                 goto redo_sifflags;
1264                         }
1265                         rc = cxgbe_uninit_synchronized(pi);
1266                 }
1267                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1268                 break;
1269
1270         case SIOCADDMULTI:
1271         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1272                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1273                 if (rc)
1274                         return (rc);
1275                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1276                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1277                 end_synchronized_op(sc, LOCK_HELD);
1278                 break;
1279
1280         case SIOCSIFCAP:
1281                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1282                 if (rc)
1283                         return (rc);
1284
1285                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1286                 if (mask & IFCAP_TXCSUM) {
1287                         ifp->if_capenable ^= IFCAP_TXCSUM;
1288                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1289
1290                         if (IFCAP_TSO4 & ifp->if_capenable &&
1291                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1292                                 ifp->if_capenable &= ~IFCAP_TSO4;
1293                                 if_printf(ifp,
1294                                     "tso4 disabled due to -txcsum.\n");
1295                         }
1296                 }
1297                 if (mask & IFCAP_TXCSUM_IPV6) {
1298                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1299                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1300
1301                         if (IFCAP_TSO6 & ifp->if_capenable &&
1302                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1303                                 ifp->if_capenable &= ~IFCAP_TSO6;
1304                                 if_printf(ifp,
1305                                     "tso6 disabled due to -txcsum6.\n");
1306                         }
1307                 }
1308                 if (mask & IFCAP_RXCSUM)
1309                         ifp->if_capenable ^= IFCAP_RXCSUM;
1310                 if (mask & IFCAP_RXCSUM_IPV6)
1311                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1312
1313                 /*
1314                  * Note that we leave CSUM_TSO alone (it is always set).  The
1315                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1316                  * sending a TSO request our way, so it's sufficient to toggle
1317                  * IFCAP_TSOx only.
1318                  */
1319                 if (mask & IFCAP_TSO4) {
1320                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1321                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1322                                 if_printf(ifp, "enable txcsum first.\n");
1323                                 rc = EAGAIN;
1324                                 goto fail;
1325                         }
1326                         ifp->if_capenable ^= IFCAP_TSO4;
1327                 }
1328                 if (mask & IFCAP_TSO6) {
1329                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1330                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1331                                 if_printf(ifp, "enable txcsum6 first.\n");
1332                                 rc = EAGAIN;
1333                                 goto fail;
1334                         }
1335                         ifp->if_capenable ^= IFCAP_TSO6;
1336                 }
1337                 if (mask & IFCAP_LRO) {
1338 #if defined(INET) || defined(INET6)
1339                         int i;
1340                         struct sge_rxq *rxq;
1341
1342                         ifp->if_capenable ^= IFCAP_LRO;
1343                         for_each_rxq(pi, i, rxq) {
1344                                 if (ifp->if_capenable & IFCAP_LRO)
1345                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1346                                 else
1347                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1348                         }
1349 #endif
1350                 }
1351 #ifdef TCP_OFFLOAD
1352                 if (mask & IFCAP_TOE) {
1353                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1354
1355                         rc = toe_capability(pi, enable);
1356                         if (rc != 0)
1357                                 goto fail;
1358
1359                         ifp->if_capenable ^= mask;
1360                 }
1361 #endif
1362                 if (mask & IFCAP_VLAN_HWTAGGING) {
1363                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1364                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1365                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1366                 }
1367                 if (mask & IFCAP_VLAN_MTU) {
1368                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1369
1370                         /* Need to find out how to disable auto-mtu-inflation */
1371                 }
1372                 if (mask & IFCAP_VLAN_HWTSO)
1373                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1374                 if (mask & IFCAP_VLAN_HWCSUM)
1375                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1376
1377 #ifdef VLAN_CAPABILITIES
1378                 VLAN_CAPABILITIES(ifp);
1379 #endif
1380 fail:
1381                 end_synchronized_op(sc, 0);
1382                 break;
1383
1384         case SIOCSIFMEDIA:
1385         case SIOCGIFMEDIA:
1386                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1387                 break;
1388
1389         default:
1390                 rc = ether_ioctl(ifp, cmd, data);
1391         }
1392
1393         return (rc);
1394 }
1395
1396 static int
1397 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1398 {
1399         struct port_info *pi = ifp->if_softc;
1400         struct adapter *sc = pi->adapter;
1401         struct sge_txq *txq;
1402         void *items[1];
1403         int rc;
1404
1405         M_ASSERTPKTHDR(m);
1406         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1407
1408         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1409                 m_freem(m);
1410                 return (ENETDOWN);
1411         }
1412
1413         rc = parse_pkt(&m);
1414         if (__predict_false(rc != 0)) {
1415                 MPASS(m == NULL);                       /* was freed already */
1416                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1417                 return (rc);
1418         }
1419
1420         /* Select a txq. */
1421         txq = &sc->sge.txq[pi->first_txq];
1422         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1423                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1424                     pi->rsrv_noflowq);
1425
1426         items[0] = m;
1427         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1428         if (__predict_false(rc != 0))
1429                 m_freem(m);
1430
1431         return (rc);
1432 }
1433
1434 static void
1435 cxgbe_qflush(struct ifnet *ifp)
1436 {
1437         struct port_info *pi = ifp->if_softc;
1438         struct sge_txq *txq;
1439         int i;
1440
1441         /* queues do not exist if !PORT_INIT_DONE. */
1442         if (pi->flags & PORT_INIT_DONE) {
1443                 for_each_txq(pi, i, txq) {
1444                         TXQ_LOCK(txq);
1445                         txq->eq.flags &= ~EQ_ENABLED;
1446                         TXQ_UNLOCK(txq);
1447                         while (!mp_ring_is_idle(txq->r)) {
1448                                 mp_ring_check_drainage(txq->r, 0);
1449                                 pause("qflush", 1);
1450                         }
1451                 }
1452         }
1453         if_qflush(ifp);
1454 }
1455
1456 static int
1457 cxgbe_media_change(struct ifnet *ifp)
1458 {
1459         struct port_info *pi = ifp->if_softc;
1460
1461         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1462
1463         return (EOPNOTSUPP);
1464 }
1465
1466 static void
1467 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1468 {
1469         struct port_info *pi = ifp->if_softc;
1470         struct ifmedia *media = NULL;
1471         struct ifmedia_entry *cur;
1472         int speed = pi->link_cfg.speed;
1473 #ifdef INVARIANTS
1474         int data = (pi->port_type << 8) | pi->mod_type;
1475 #endif
1476
1477         if (ifp == pi->ifp)
1478                 media = &pi->media;
1479 #ifdef DEV_NETMAP
1480         else if (ifp == pi->nm_ifp)
1481                 media = &pi->nm_media;
1482 #endif
1483         MPASS(media != NULL);
1484
1485         cur = media->ifm_cur;
1486         MPASS(cur->ifm_data == data);
1487
1488         ifmr->ifm_status = IFM_AVALID;
1489         if (!pi->link_cfg.link_ok)
1490                 return;
1491
1492         ifmr->ifm_status |= IFM_ACTIVE;
1493
1494         /* active and current will differ iff current media is autoselect. */
1495         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1496                 return;
1497
1498         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1499         if (speed == SPEED_10000)
1500                 ifmr->ifm_active |= IFM_10G_T;
1501         else if (speed == SPEED_1000)
1502                 ifmr->ifm_active |= IFM_1000_T;
1503         else if (speed == SPEED_100)
1504                 ifmr->ifm_active |= IFM_100_TX;
1505         else if (speed == SPEED_10)
1506                 ifmr->ifm_active |= IFM_10_T;
1507         else
1508                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1509                             speed));
1510 }
1511
1512 void
1513 t4_fatal_err(struct adapter *sc)
1514 {
1515         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1516         t4_intr_disable(sc);
1517         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1518             device_get_nameunit(sc->dev));
1519 }
1520
1521 static int
1522 map_bars_0_and_4(struct adapter *sc)
1523 {
1524         sc->regs_rid = PCIR_BAR(0);
1525         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1526             &sc->regs_rid, RF_ACTIVE);
1527         if (sc->regs_res == NULL) {
1528                 device_printf(sc->dev, "cannot map registers.\n");
1529                 return (ENXIO);
1530         }
1531         sc->bt = rman_get_bustag(sc->regs_res);
1532         sc->bh = rman_get_bushandle(sc->regs_res);
1533         sc->mmio_len = rman_get_size(sc->regs_res);
1534         setbit(&sc->doorbells, DOORBELL_KDB);
1535
1536         sc->msix_rid = PCIR_BAR(4);
1537         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1538             &sc->msix_rid, RF_ACTIVE);
1539         if (sc->msix_res == NULL) {
1540                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1541                 return (ENXIO);
1542         }
1543
1544         return (0);
1545 }
1546
1547 static int
1548 map_bar_2(struct adapter *sc)
1549 {
1550
1551         /*
1552          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1553          * to map it if RDMA is disabled.
1554          */
1555         if (is_t4(sc) && sc->rdmacaps == 0)
1556                 return (0);
1557
1558         sc->udbs_rid = PCIR_BAR(2);
1559         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1560             &sc->udbs_rid, RF_ACTIVE);
1561         if (sc->udbs_res == NULL) {
1562                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1563                 return (ENXIO);
1564         }
1565         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1566
1567         if (is_t5(sc)) {
1568                 setbit(&sc->doorbells, DOORBELL_UDB);
1569 #if defined(__i386__) || defined(__amd64__)
1570                 if (t5_write_combine) {
1571                         int rc;
1572
1573                         /*
1574                          * Enable write combining on BAR2.  This is the
1575                          * userspace doorbell BAR and is split into 128B
1576                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1577                          * with an egress queue.  The first 64B has the doorbell
1578                          * and the second 64B can be used to submit a tx work
1579                          * request with an implicit doorbell.
1580                          */
1581
1582                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1583                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1584                         if (rc == 0) {
1585                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1586                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1587                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1588                         } else {
1589                                 device_printf(sc->dev,
1590                                     "couldn't enable write combining: %d\n",
1591                                     rc);
1592                         }
1593
1594                         t4_write_reg(sc, A_SGE_STAT_CFG,
1595                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1596                 }
1597 #endif
1598         }
1599
1600         return (0);
1601 }
1602
1603 static const struct memwin t4_memwin[] = {
1604         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1605         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1606         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1607 };
1608
1609 static const struct memwin t5_memwin[] = {
1610         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1611         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1612         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1613 };
1614
1615 static void
1616 setup_memwin(struct adapter *sc)
1617 {
1618         const struct memwin *mw;
1619         int i, n;
1620         uint32_t bar0;
1621
1622         if (is_t4(sc)) {
1623                 /*
1624                  * Read low 32b of bar0 indirectly via the hardware backdoor
1625                  * mechanism.  Works from within PCI passthrough environments
1626                  * too, where rman_get_start() can return a different value.  We
1627                  * need to program the T4 memory window decoders with the actual
1628                  * addresses that will be coming across the PCIe link.
1629                  */
1630                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1631                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1632
1633                 mw = &t4_memwin[0];
1634                 n = nitems(t4_memwin);
1635         } else {
1636                 /* T5 uses the relative offset inside the PCIe BAR */
1637                 bar0 = 0;
1638
1639                 mw = &t5_memwin[0];
1640                 n = nitems(t5_memwin);
1641         }
1642
1643         for (i = 0; i < n; i++, mw++) {
1644                 t4_write_reg(sc,
1645                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1646                     (mw->base + bar0) | V_BIR(0) |
1647                     V_WINDOW(ilog2(mw->aperture) - 10));
1648         }
1649
1650         /* flush */
1651         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1652 }
1653
1654 /*
1655  * Verify that the memory range specified by the addr/len pair is valid and lies
1656  * entirely within a single region (EDCx or MCx).
1657  */
1658 static int
1659 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1660 {
1661         uint32_t em, addr_len, maddr, mlen;
1662
1663         /* Memory can only be accessed in naturally aligned 4 byte units */
1664         if (addr & 3 || len & 3 || len == 0)
1665                 return (EINVAL);
1666
1667         /* Enabled memories */
1668         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1669         if (em & F_EDRAM0_ENABLE) {
1670                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1671                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1672                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1673                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1674                     addr + len <= maddr + mlen)
1675                         return (0);
1676         }
1677         if (em & F_EDRAM1_ENABLE) {
1678                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1679                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1680                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1681                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1682                     addr + len <= maddr + mlen)
1683                         return (0);
1684         }
1685         if (em & F_EXT_MEM_ENABLE) {
1686                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1687                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1688                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1689                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1690                     addr + len <= maddr + mlen)
1691                         return (0);
1692         }
1693         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1694                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1695                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1696                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1697                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1698                     addr + len <= maddr + mlen)
1699                         return (0);
1700         }
1701
1702         return (EFAULT);
1703 }
1704
1705 static int
1706 fwmtype_to_hwmtype(int mtype)
1707 {
1708
1709         switch (mtype) {
1710         case FW_MEMTYPE_EDC0:
1711                 return (MEM_EDC0);
1712         case FW_MEMTYPE_EDC1:
1713                 return (MEM_EDC1);
1714         case FW_MEMTYPE_EXTMEM:
1715                 return (MEM_MC0);
1716         case FW_MEMTYPE_EXTMEM1:
1717                 return (MEM_MC1);
1718         default:
1719                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1720         }
1721 }
1722
1723 /*
1724  * Verify that the memory range specified by the memtype/offset/len pair is
1725  * valid and lies entirely within the memtype specified.  The global address of
1726  * the start of the range is returned in addr.
1727  */
1728 static int
1729 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1730     uint32_t *addr)
1731 {
1732         uint32_t em, addr_len, maddr, mlen;
1733
1734         /* Memory can only be accessed in naturally aligned 4 byte units */
1735         if (off & 3 || len & 3 || len == 0)
1736                 return (EINVAL);
1737
1738         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1739         switch (fwmtype_to_hwmtype(mtype)) {
1740         case MEM_EDC0:
1741                 if (!(em & F_EDRAM0_ENABLE))
1742                         return (EINVAL);
1743                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1744                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1745                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1746                 break;
1747         case MEM_EDC1:
1748                 if (!(em & F_EDRAM1_ENABLE))
1749                         return (EINVAL);
1750                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1751                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1752                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1753                 break;
1754         case MEM_MC:
1755                 if (!(em & F_EXT_MEM_ENABLE))
1756                         return (EINVAL);
1757                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1758                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1759                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1760                 break;
1761         case MEM_MC1:
1762                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1763                         return (EINVAL);
1764                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1765                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1766                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1767                 break;
1768         default:
1769                 return (EINVAL);
1770         }
1771
1772         if (mlen > 0 && off < mlen && off + len <= mlen) {
1773                 *addr = maddr + off;    /* global address */
1774                 return (0);
1775         }
1776
1777         return (EFAULT);
1778 }
1779
1780 static void
1781 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1782 {
1783         const struct memwin *mw;
1784
1785         if (is_t4(sc)) {
1786                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1787                     ("%s: incorrect memwin# (%d)", __func__, win));
1788                 mw = &t4_memwin[win];
1789         } else {
1790                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1791                     ("%s: incorrect memwin# (%d)", __func__, win));
1792                 mw = &t5_memwin[win];
1793         }
1794
1795         if (base != NULL)
1796                 *base = mw->base;
1797         if (aperture != NULL)
1798                 *aperture = mw->aperture;
1799 }
1800
1801 /*
1802  * Positions the memory window such that it can be used to access the specified
1803  * address in the chip's address space.  The return value is the offset of addr
1804  * from the start of the window.
1805  */
1806 static uint32_t
1807 position_memwin(struct adapter *sc, int n, uint32_t addr)
1808 {
1809         uint32_t start, pf;
1810         uint32_t reg;
1811
1812         KASSERT(n >= 0 && n <= 3,
1813             ("%s: invalid window %d.", __func__, n));
1814         KASSERT((addr & 3) == 0,
1815             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1816
1817         if (is_t4(sc)) {
1818                 pf = 0;
1819                 start = addr & ~0xf;    /* start must be 16B aligned */
1820         } else {
1821                 pf = V_PFNUM(sc->pf);
1822                 start = addr & ~0x7f;   /* start must be 128B aligned */
1823         }
1824         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1825
1826         t4_write_reg(sc, reg, start | pf);
1827         t4_read_reg(sc, reg);
1828
1829         return (addr - start);
1830 }
1831
1832 static int
1833 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1834     struct intrs_and_queues *iaq)
1835 {
1836         int rc, itype, navail, nrxq10g, nrxq1g, n;
1837         int nofldrxq10g = 0, nofldrxq1g = 0;
1838         int nnmrxq10g = 0, nnmrxq1g = 0;
1839
1840         bzero(iaq, sizeof(*iaq));
1841
1842         iaq->ntxq10g = t4_ntxq10g;
1843         iaq->ntxq1g = t4_ntxq1g;
1844         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1845         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1846         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1847 #ifdef TCP_OFFLOAD
1848         if (is_offload(sc)) {
1849                 iaq->nofldtxq10g = t4_nofldtxq10g;
1850                 iaq->nofldtxq1g = t4_nofldtxq1g;
1851                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1852                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1853         }
1854 #endif
1855 #ifdef DEV_NETMAP
1856         iaq->nnmtxq10g = t4_nnmtxq10g;
1857         iaq->nnmtxq1g = t4_nnmtxq1g;
1858         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1859         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1860 #endif
1861
1862         for (itype = INTR_MSIX; itype; itype >>= 1) {
1863
1864                 if ((itype & t4_intr_types) == 0)
1865                         continue;       /* not allowed */
1866
1867                 if (itype == INTR_MSIX)
1868                         navail = pci_msix_count(sc->dev);
1869                 else if (itype == INTR_MSI)
1870                         navail = pci_msi_count(sc->dev);
1871                 else
1872                         navail = 1;
1873 restart:
1874                 if (navail == 0)
1875                         continue;
1876
1877                 iaq->intr_type = itype;
1878                 iaq->intr_flags_10g = 0;
1879                 iaq->intr_flags_1g = 0;
1880
1881                 /*
1882                  * Best option: an interrupt vector for errors, one for the
1883                  * firmware event queue, and one for every rxq (NIC, TOE, and
1884                  * netmap).
1885                  */
1886                 iaq->nirq = T4_EXTRA_INTR;
1887                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1888                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1889                 if (iaq->nirq <= navail &&
1890                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1891                         iaq->intr_flags_10g = INTR_ALL;
1892                         iaq->intr_flags_1g = INTR_ALL;
1893                         goto allocate;
1894                 }
1895
1896                 /*
1897                  * Second best option: a vector for errors, one for the firmware
1898                  * event queue, and vectors for either all the NIC rx queues or
1899                  * all the TOE rx queues.  The queues that don't get vectors
1900                  * will forward their interrupts to those that do.
1901                  *
1902                  * Note: netmap rx queues cannot be created early and so they
1903                  * can't be setup to receive forwarded interrupts for others.
1904                  */
1905                 iaq->nirq = T4_EXTRA_INTR;
1906                 if (nrxq10g >= nofldrxq10g) {
1907                         iaq->intr_flags_10g = INTR_RXQ;
1908                         iaq->nirq += n10g * nrxq10g;
1909 #ifdef DEV_NETMAP
1910                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1911 #endif
1912                 } else {
1913                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1914                         iaq->nirq += n10g * nofldrxq10g;
1915 #ifdef DEV_NETMAP
1916                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1917 #endif
1918                 }
1919                 if (nrxq1g >= nofldrxq1g) {
1920                         iaq->intr_flags_1g = INTR_RXQ;
1921                         iaq->nirq += n1g * nrxq1g;
1922 #ifdef DEV_NETMAP
1923                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1924 #endif
1925                 } else {
1926                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1927                         iaq->nirq += n1g * nofldrxq1g;
1928 #ifdef DEV_NETMAP
1929                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1930 #endif
1931                 }
1932                 if (iaq->nirq <= navail &&
1933                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1934                         goto allocate;
1935
1936                 /*
1937                  * Next best option: an interrupt vector for errors, one for the
1938                  * firmware event queue, and at least one per port.  At this
1939                  * point we know we'll have to downsize nrxq and/or nofldrxq
1940                  * and/or nnmrxq to fit what's available to us.
1941                  */
1942                 iaq->nirq = T4_EXTRA_INTR;
1943                 iaq->nirq += n10g + n1g;
1944                 if (iaq->nirq <= navail) {
1945                         int leftover = navail - iaq->nirq;
1946
1947                         if (n10g > 0) {
1948                                 int target = max(nrxq10g, nofldrxq10g);
1949
1950                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1951                                     INTR_RXQ : INTR_OFLD_RXQ;
1952
1953                                 n = 1;
1954                                 while (n < target && leftover >= n10g) {
1955                                         leftover -= n10g;
1956                                         iaq->nirq += n10g;
1957                                         n++;
1958                                 }
1959                                 iaq->nrxq10g = min(n, nrxq10g);
1960 #ifdef TCP_OFFLOAD
1961                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1962 #endif
1963 #ifdef DEV_NETMAP
1964                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1965 #endif
1966                         }
1967
1968                         if (n1g > 0) {
1969                                 int target = max(nrxq1g, nofldrxq1g);
1970
1971                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1972                                     INTR_RXQ : INTR_OFLD_RXQ;
1973
1974                                 n = 1;
1975                                 while (n < target && leftover >= n1g) {
1976                                         leftover -= n1g;
1977                                         iaq->nirq += n1g;
1978                                         n++;
1979                                 }
1980                                 iaq->nrxq1g = min(n, nrxq1g);
1981 #ifdef TCP_OFFLOAD
1982                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1983 #endif
1984 #ifdef DEV_NETMAP
1985                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1986 #endif
1987                         }
1988
1989                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1990                                 goto allocate;
1991                 }
1992
1993                 /*
1994                  * Least desirable option: one interrupt vector for everything.
1995                  */
1996                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1997                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
1998 #ifdef TCP_OFFLOAD
1999                 if (is_offload(sc))
2000                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2001 #endif
2002 #ifdef DEV_NETMAP
2003                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2004 #endif
2005
2006 allocate:
2007                 navail = iaq->nirq;
2008                 rc = 0;
2009                 if (itype == INTR_MSIX)
2010                         rc = pci_alloc_msix(sc->dev, &navail);
2011                 else if (itype == INTR_MSI)
2012                         rc = pci_alloc_msi(sc->dev, &navail);
2013
2014                 if (rc == 0) {
2015                         if (navail == iaq->nirq)
2016                                 return (0);
2017
2018                         /*
2019                          * Didn't get the number requested.  Use whatever number
2020                          * the kernel is willing to allocate (it's in navail).
2021                          */
2022                         device_printf(sc->dev, "fewer vectors than requested, "
2023                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2024                             itype, iaq->nirq, navail);
2025                         pci_release_msi(sc->dev);
2026                         goto restart;
2027                 }
2028
2029                 device_printf(sc->dev,
2030                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2031                     itype, rc, iaq->nirq, navail);
2032         }
2033
2034         device_printf(sc->dev,
2035             "failed to find a usable interrupt type.  "
2036             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2037             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2038
2039         return (ENXIO);
2040 }
2041
2042 #define FW_VERSION(chip) ( \
2043     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2044     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2045     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2046     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2047 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2048
2049 struct fw_info {
2050         uint8_t chip;
2051         char *kld_name;
2052         char *fw_mod_name;
2053         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2054 } fw_info[] = {
2055         {
2056                 .chip = CHELSIO_T4,
2057                 .kld_name = "t4fw_cfg",
2058                 .fw_mod_name = "t4fw",
2059                 .fw_hdr = {
2060                         .chip = FW_HDR_CHIP_T4,
2061                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2062                         .intfver_nic = FW_INTFVER(T4, NIC),
2063                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2064                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2065                         .intfver_ri = FW_INTFVER(T4, RI),
2066                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2067                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2068                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2069                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2070                 },
2071         }, {
2072                 .chip = CHELSIO_T5,
2073                 .kld_name = "t5fw_cfg",
2074                 .fw_mod_name = "t5fw",
2075                 .fw_hdr = {
2076                         .chip = FW_HDR_CHIP_T5,
2077                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2078                         .intfver_nic = FW_INTFVER(T5, NIC),
2079                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2080                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2081                         .intfver_ri = FW_INTFVER(T5, RI),
2082                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2083                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2084                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2085                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2086                 },
2087         }
2088 };
2089
2090 static struct fw_info *
2091 find_fw_info(int chip)
2092 {
2093         int i;
2094
2095         for (i = 0; i < nitems(fw_info); i++) {
2096                 if (fw_info[i].chip == chip)
2097                         return (&fw_info[i]);
2098         }
2099         return (NULL);
2100 }
2101
2102 /*
2103  * Is the given firmware API compatible with the one the driver was compiled
2104  * with?
2105  */
2106 static int
2107 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2108 {
2109
2110         /* short circuit if it's the exact same firmware version */
2111         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2112                 return (1);
2113
2114         /*
2115          * XXX: Is this too conservative?  Perhaps I should limit this to the
2116          * features that are supported in the driver.
2117          */
2118 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2119         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2120             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2121             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2122                 return (1);
2123 #undef SAME_INTF
2124
2125         return (0);
2126 }
2127
2128 /*
2129  * The firmware in the KLD is usable, but should it be installed?  This routine
2130  * explains itself in detail if it indicates the KLD firmware should be
2131  * installed.
2132  */
2133 static int
2134 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2135 {
2136         const char *reason;
2137
2138         if (!card_fw_usable) {
2139                 reason = "incompatible or unusable";
2140                 goto install;
2141         }
2142
2143         if (k > c) {
2144                 reason = "older than the version bundled with this driver";
2145                 goto install;
2146         }
2147
2148         if (t4_fw_install == 2 && k != c) {
2149                 reason = "different than the version bundled with this driver";
2150                 goto install;
2151         }
2152
2153         return (0);
2154
2155 install:
2156         if (t4_fw_install == 0) {
2157                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2158                     "but the driver is prohibited from installing a different "
2159                     "firmware on the card.\n",
2160                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2161                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2162
2163                 return (0);
2164         }
2165
2166         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2167             "installing firmware %u.%u.%u.%u on card.\n",
2168             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2169             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2170             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2171             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2172
2173         return (1);
2174 }
2175 /*
2176  * Establish contact with the firmware and determine if we are the master driver
2177  * or not, and whether we are responsible for chip initialization.
2178  */
2179 static int
2180 prep_firmware(struct adapter *sc)
2181 {
2182         const struct firmware *fw = NULL, *default_cfg;
2183         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2184         enum dev_state state;
2185         struct fw_info *fw_info;
2186         struct fw_hdr *card_fw;         /* fw on the card */
2187         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2188         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2189                                            against */
2190
2191         /* Contact firmware. */
2192         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2193         if (rc < 0 || state == DEV_STATE_ERR) {
2194                 rc = -rc;
2195                 device_printf(sc->dev,
2196                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2197                 return (rc);
2198         }
2199         pf = rc;
2200         if (pf == sc->mbox)
2201                 sc->flags |= MASTER_PF;
2202         else if (state == DEV_STATE_UNINIT) {
2203                 /*
2204                  * We didn't get to be the master so we definitely won't be
2205                  * configuring the chip.  It's a bug if someone else hasn't
2206                  * configured it already.
2207                  */
2208                 device_printf(sc->dev, "couldn't be master(%d), "
2209                     "device not already initialized either(%d).\n", rc, state);
2210                 return (EDOOFUS);
2211         }
2212
2213         /* This is the firmware whose headers the driver was compiled against */
2214         fw_info = find_fw_info(chip_id(sc));
2215         if (fw_info == NULL) {
2216                 device_printf(sc->dev,
2217                     "unable to look up firmware information for chip %d.\n",
2218                     chip_id(sc));
2219                 return (EINVAL);
2220         }
2221         drv_fw = &fw_info->fw_hdr;
2222
2223         /*
2224          * The firmware KLD contains many modules.  The KLD name is also the
2225          * name of the module that contains the default config file.
2226          */
2227         default_cfg = firmware_get(fw_info->kld_name);
2228
2229         /* Read the header of the firmware on the card */
2230         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2231         rc = -t4_read_flash(sc, FLASH_FW_START,
2232             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2233         if (rc == 0)
2234                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2235         else {
2236                 device_printf(sc->dev,
2237                     "Unable to read card's firmware header: %d\n", rc);
2238                 card_fw_usable = 0;
2239         }
2240
2241         /* This is the firmware in the KLD */
2242         fw = firmware_get(fw_info->fw_mod_name);
2243         if (fw != NULL) {
2244                 kld_fw = (const void *)fw->data;
2245                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2246         } else {
2247                 kld_fw = NULL;
2248                 kld_fw_usable = 0;
2249         }
2250
2251         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2252             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2253                 /*
2254                  * Common case: the firmware on the card is an exact match and
2255                  * the KLD is an exact match too, or the KLD is
2256                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2257                  * here -- use cxgbetool loadfw if you want to reinstall the
2258                  * same firmware as the one on the card.
2259                  */
2260         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2261             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2262             be32toh(card_fw->fw_ver))) {
2263
2264                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2265                 if (rc != 0) {
2266                         device_printf(sc->dev,
2267                             "failed to install firmware: %d\n", rc);
2268                         goto done;
2269                 }
2270
2271                 /* Installed successfully, update the cached header too. */
2272                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2273                 card_fw_usable = 1;
2274                 need_fw_reset = 0;      /* already reset as part of load_fw */
2275         }
2276
2277         if (!card_fw_usable) {
2278                 uint32_t d, c, k;
2279
2280                 d = ntohl(drv_fw->fw_ver);
2281                 c = ntohl(card_fw->fw_ver);
2282                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2283
2284                 device_printf(sc->dev, "Cannot find a usable firmware: "
2285                     "fw_install %d, chip state %d, "
2286                     "driver compiled with %d.%d.%d.%d, "
2287                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2288                     t4_fw_install, state,
2289                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2290                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2291                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2292                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2293                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2294                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2295                 rc = EINVAL;
2296                 goto done;
2297         }
2298
2299         /* We're using whatever's on the card and it's known to be good. */
2300         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2301         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2302             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2303             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2304             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2305             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2306         t4_get_tp_version(sc, &sc->params.tp_vers);
2307
2308         /* Reset device */
2309         if (need_fw_reset &&
2310             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2311                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2312                 if (rc != ETIMEDOUT && rc != EIO)
2313                         t4_fw_bye(sc, sc->mbox);
2314                 goto done;
2315         }
2316         sc->flags |= FW_OK;
2317
2318         rc = get_params__pre_init(sc);
2319         if (rc != 0)
2320                 goto done; /* error message displayed already */
2321
2322         /* Partition adapter resources as specified in the config file. */
2323         if (state == DEV_STATE_UNINIT) {
2324
2325                 KASSERT(sc->flags & MASTER_PF,
2326                     ("%s: trying to change chip settings when not master.",
2327                     __func__));
2328
2329                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2330                 if (rc != 0)
2331                         goto done;      /* error message displayed already */
2332
2333                 t4_tweak_chip_settings(sc);
2334
2335                 /* get basic stuff going */
2336                 rc = -t4_fw_initialize(sc, sc->mbox);
2337                 if (rc != 0) {
2338                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2339                         goto done;
2340                 }
2341         } else {
2342                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2343                 sc->cfcsum = 0;
2344         }
2345
2346 done:
2347         free(card_fw, M_CXGBE);
2348         if (fw != NULL)
2349                 firmware_put(fw, FIRMWARE_UNLOAD);
2350         if (default_cfg != NULL)
2351                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2352
2353         return (rc);
2354 }
2355
2356 #define FW_PARAM_DEV(param) \
2357         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2358          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2359 #define FW_PARAM_PFVF(param) \
2360         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2361          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2362
2363 /*
2364  * Partition chip resources for use between various PFs, VFs, etc.
2365  */
2366 static int
2367 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2368     const char *name_prefix)
2369 {
2370         const struct firmware *cfg = NULL;
2371         int rc = 0;
2372         struct fw_caps_config_cmd caps;
2373         uint32_t mtype, moff, finicsum, cfcsum;
2374
2375         /*
2376          * Figure out what configuration file to use.  Pick the default config
2377          * file for the card if the user hasn't specified one explicitly.
2378          */
2379         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2380         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2381                 /* Card specific overrides go here. */
2382                 if (pci_get_device(sc->dev) == 0x440a)
2383                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2384                 if (is_fpga(sc))
2385                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2386         }
2387
2388         /*
2389          * We need to load another module if the profile is anything except
2390          * "default" or "flash".
2391          */
2392         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2393             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2394                 char s[32];
2395
2396                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2397                 cfg = firmware_get(s);
2398                 if (cfg == NULL) {
2399                         if (default_cfg != NULL) {
2400                                 device_printf(sc->dev,
2401                                     "unable to load module \"%s\" for "
2402                                     "configuration profile \"%s\", will use "
2403                                     "the default config file instead.\n",
2404                                     s, sc->cfg_file);
2405                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2406                                     "%s", DEFAULT_CF);
2407                         } else {
2408                                 device_printf(sc->dev,
2409                                     "unable to load module \"%s\" for "
2410                                     "configuration profile \"%s\", will use "
2411                                     "the config file on the card's flash "
2412                                     "instead.\n", s, sc->cfg_file);
2413                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2414                                     "%s", FLASH_CF);
2415                         }
2416                 }
2417         }
2418
2419         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2420             default_cfg == NULL) {
2421                 device_printf(sc->dev,
2422                     "default config file not available, will use the config "
2423                     "file on the card's flash instead.\n");
2424                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2425         }
2426
2427         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2428                 u_int cflen, i, n;
2429                 const uint32_t *cfdata;
2430                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2431
2432                 KASSERT(cfg != NULL || default_cfg != NULL,
2433                     ("%s: no config to upload", __func__));
2434
2435                 /*
2436                  * Ask the firmware where it wants us to upload the config file.
2437                  */
2438                 param = FW_PARAM_DEV(CF);
2439                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2440                 if (rc != 0) {
2441                         /* No support for config file?  Shouldn't happen. */
2442                         device_printf(sc->dev,
2443                             "failed to query config file location: %d.\n", rc);
2444                         goto done;
2445                 }
2446                 mtype = G_FW_PARAMS_PARAM_Y(val);
2447                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2448
2449                 /*
2450                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2451                  * useless stuffing/comments at the end of the config file so
2452                  * it's ok to simply throw away the last remaining bytes when
2453                  * the config file is not an exact multiple of 4.  This also
2454                  * helps with the validate_mt_off_len check.
2455                  */
2456                 if (cfg != NULL) {
2457                         cflen = cfg->datasize & ~3;
2458                         cfdata = cfg->data;
2459                 } else {
2460                         cflen = default_cfg->datasize & ~3;
2461                         cfdata = default_cfg->data;
2462                 }
2463
2464                 if (cflen > FLASH_CFG_MAX_SIZE) {
2465                         device_printf(sc->dev,
2466                             "config file too long (%d, max allowed is %d).  "
2467                             "Will try to use the config on the card, if any.\n",
2468                             cflen, FLASH_CFG_MAX_SIZE);
2469                         goto use_config_on_flash;
2470                 }
2471
2472                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2473                 if (rc != 0) {
2474                         device_printf(sc->dev,
2475                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2476                             "Will try to use the config on the card, if any.\n",
2477                             __func__, mtype, moff, cflen, rc);
2478                         goto use_config_on_flash;
2479                 }
2480
2481                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2482                 while (cflen) {
2483                         off = position_memwin(sc, 2, addr);
2484                         n = min(cflen, mw_aperture - off);
2485                         for (i = 0; i < n; i += 4)
2486                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2487                         cflen -= n;
2488                         addr += n;
2489                 }
2490         } else {
2491 use_config_on_flash:
2492                 mtype = FW_MEMTYPE_FLASH;
2493                 moff = t4_flash_cfg_addr(sc);
2494         }
2495
2496         bzero(&caps, sizeof(caps));
2497         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2498             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2499         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2500             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2501             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2502         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2503         if (rc != 0) {
2504                 device_printf(sc->dev,
2505                     "failed to pre-process config file: %d "
2506                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2507                 goto done;
2508         }
2509
2510         finicsum = be32toh(caps.finicsum);
2511         cfcsum = be32toh(caps.cfcsum);
2512         if (finicsum != cfcsum) {
2513                 device_printf(sc->dev,
2514                     "WARNING: config file checksum mismatch: %08x %08x\n",
2515                     finicsum, cfcsum);
2516         }
2517         sc->cfcsum = cfcsum;
2518
2519 #define LIMIT_CAPS(x) do { \
2520         caps.x &= htobe16(t4_##x##_allowed); \
2521 } while (0)
2522
2523         /*
2524          * Let the firmware know what features will (not) be used so it can tune
2525          * things accordingly.
2526          */
2527         LIMIT_CAPS(linkcaps);
2528         LIMIT_CAPS(niccaps);
2529         LIMIT_CAPS(toecaps);
2530         LIMIT_CAPS(rdmacaps);
2531         LIMIT_CAPS(iscsicaps);
2532         LIMIT_CAPS(fcoecaps);
2533 #undef LIMIT_CAPS
2534
2535         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2536             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2537         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2538         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2539         if (rc != 0) {
2540                 device_printf(sc->dev,
2541                     "failed to process config file: %d.\n", rc);
2542         }
2543 done:
2544         if (cfg != NULL)
2545                 firmware_put(cfg, FIRMWARE_UNLOAD);
2546         return (rc);
2547 }
2548
2549 /*
2550  * Retrieve parameters that are needed (or nice to have) very early.
2551  */
2552 static int
2553 get_params__pre_init(struct adapter *sc)
2554 {
2555         int rc;
2556         uint32_t param[2], val[2];
2557         struct fw_devlog_cmd cmd;
2558         struct devlog_params *dlog = &sc->params.devlog;
2559
2560         param[0] = FW_PARAM_DEV(PORTVEC);
2561         param[1] = FW_PARAM_DEV(CCLK);
2562         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2563         if (rc != 0) {
2564                 device_printf(sc->dev,
2565                     "failed to query parameters (pre_init): %d.\n", rc);
2566                 return (rc);
2567         }
2568
2569         sc->params.portvec = val[0];
2570         sc->params.nports = bitcount32(val[0]);
2571         sc->params.vpd.cclk = val[1];
2572
2573         /* Read device log parameters. */
2574         bzero(&cmd, sizeof(cmd));
2575         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2576             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2577         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2578         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2579         if (rc != 0) {
2580                 device_printf(sc->dev,
2581                     "failed to get devlog parameters: %d.\n", rc);
2582                 bzero(dlog, sizeof (*dlog));
2583                 rc = 0; /* devlog isn't critical for device operation */
2584         } else {
2585                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2586                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2587                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2588                 dlog->size = be32toh(cmd.memsize_devlog);
2589         }
2590
2591         return (rc);
2592 }
2593
2594 /*
2595  * Retrieve various parameters that are of interest to the driver.  The device
2596  * has been initialized by the firmware at this point.
2597  */
2598 static int
2599 get_params__post_init(struct adapter *sc)
2600 {
2601         int rc;
2602         uint32_t param[7], val[7];
2603         struct fw_caps_config_cmd caps;
2604
2605         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2606         param[1] = FW_PARAM_PFVF(EQ_START);
2607         param[2] = FW_PARAM_PFVF(FILTER_START);
2608         param[3] = FW_PARAM_PFVF(FILTER_END);
2609         param[4] = FW_PARAM_PFVF(L2T_START);
2610         param[5] = FW_PARAM_PFVF(L2T_END);
2611         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2612         if (rc != 0) {
2613                 device_printf(sc->dev,
2614                     "failed to query parameters (post_init): %d.\n", rc);
2615                 return (rc);
2616         }
2617
2618         sc->sge.iq_start = val[0];
2619         sc->sge.eq_start = val[1];
2620         sc->tids.ftid_base = val[2];
2621         sc->tids.nftids = val[3] - val[2] + 1;
2622         sc->params.ftid_min = val[2];
2623         sc->params.ftid_max = val[3];
2624         sc->vres.l2t.start = val[4];
2625         sc->vres.l2t.size = val[5] - val[4] + 1;
2626         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2627             ("%s: L2 table size (%u) larger than expected (%u)",
2628             __func__, sc->vres.l2t.size, L2T_SIZE));
2629
2630         /* get capabilites */
2631         bzero(&caps, sizeof(caps));
2632         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2633             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2634         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2635         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2636         if (rc != 0) {
2637                 device_printf(sc->dev,
2638                     "failed to get card capabilities: %d.\n", rc);
2639                 return (rc);
2640         }
2641
2642 #define READ_CAPS(x) do { \
2643         sc->x = htobe16(caps.x); \
2644 } while (0)
2645         READ_CAPS(linkcaps);
2646         READ_CAPS(niccaps);
2647         READ_CAPS(toecaps);
2648         READ_CAPS(rdmacaps);
2649         READ_CAPS(iscsicaps);
2650         READ_CAPS(fcoecaps);
2651
2652         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2653                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2654                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2655                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2656                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2657                 if (rc != 0) {
2658                         device_printf(sc->dev,
2659                             "failed to query NIC parameters: %d.\n", rc);
2660                         return (rc);
2661                 }
2662                 sc->tids.etid_base = val[0];
2663                 sc->params.etid_min = val[0];
2664                 sc->tids.netids = val[1] - val[0] + 1;
2665                 sc->params.netids = sc->tids.netids;
2666                 sc->params.eo_wr_cred = val[2];
2667                 sc->params.ethoffload = 1;
2668         }
2669
2670         if (sc->toecaps) {
2671                 /* query offload-related parameters */
2672                 param[0] = FW_PARAM_DEV(NTID);
2673                 param[1] = FW_PARAM_PFVF(SERVER_START);
2674                 param[2] = FW_PARAM_PFVF(SERVER_END);
2675                 param[3] = FW_PARAM_PFVF(TDDP_START);
2676                 param[4] = FW_PARAM_PFVF(TDDP_END);
2677                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2678                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2679                 if (rc != 0) {
2680                         device_printf(sc->dev,
2681                             "failed to query TOE parameters: %d.\n", rc);
2682                         return (rc);
2683                 }
2684                 sc->tids.ntids = val[0];
2685                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2686                 sc->tids.stid_base = val[1];
2687                 sc->tids.nstids = val[2] - val[1] + 1;
2688                 sc->vres.ddp.start = val[3];
2689                 sc->vres.ddp.size = val[4] - val[3] + 1;
2690                 sc->params.ofldq_wr_cred = val[5];
2691                 sc->params.offload = 1;
2692         }
2693         if (sc->rdmacaps) {
2694                 param[0] = FW_PARAM_PFVF(STAG_START);
2695                 param[1] = FW_PARAM_PFVF(STAG_END);
2696                 param[2] = FW_PARAM_PFVF(RQ_START);
2697                 param[3] = FW_PARAM_PFVF(RQ_END);
2698                 param[4] = FW_PARAM_PFVF(PBL_START);
2699                 param[5] = FW_PARAM_PFVF(PBL_END);
2700                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2701                 if (rc != 0) {
2702                         device_printf(sc->dev,
2703                             "failed to query RDMA parameters(1): %d.\n", rc);
2704                         return (rc);
2705                 }
2706                 sc->vres.stag.start = val[0];
2707                 sc->vres.stag.size = val[1] - val[0] + 1;
2708                 sc->vres.rq.start = val[2];
2709                 sc->vres.rq.size = val[3] - val[2] + 1;
2710                 sc->vres.pbl.start = val[4];
2711                 sc->vres.pbl.size = val[5] - val[4] + 1;
2712
2713                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2714                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2715                 param[2] = FW_PARAM_PFVF(CQ_START);
2716                 param[3] = FW_PARAM_PFVF(CQ_END);
2717                 param[4] = FW_PARAM_PFVF(OCQ_START);
2718                 param[5] = FW_PARAM_PFVF(OCQ_END);
2719                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2720                 if (rc != 0) {
2721                         device_printf(sc->dev,
2722                             "failed to query RDMA parameters(2): %d.\n", rc);
2723                         return (rc);
2724                 }
2725                 sc->vres.qp.start = val[0];
2726                 sc->vres.qp.size = val[1] - val[0] + 1;
2727                 sc->vres.cq.start = val[2];
2728                 sc->vres.cq.size = val[3] - val[2] + 1;
2729                 sc->vres.ocq.start = val[4];
2730                 sc->vres.ocq.size = val[5] - val[4] + 1;
2731         }
2732         if (sc->iscsicaps) {
2733                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2734                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2735                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2736                 if (rc != 0) {
2737                         device_printf(sc->dev,
2738                             "failed to query iSCSI parameters: %d.\n", rc);
2739                         return (rc);
2740                 }
2741                 sc->vres.iscsi.start = val[0];
2742                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2743         }
2744
2745         /*
2746          * We've got the params we wanted to query via the firmware.  Now grab
2747          * some others directly from the chip.
2748          */
2749         rc = t4_read_chip_settings(sc);
2750
2751         return (rc);
2752 }
2753
2754 static int
2755 set_params__post_init(struct adapter *sc)
2756 {
2757         uint32_t param, val;
2758
2759         /* ask for encapsulated CPLs */
2760         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2761         val = 1;
2762         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2763
2764         return (0);
2765 }
2766
2767 #undef FW_PARAM_PFVF
2768 #undef FW_PARAM_DEV
2769
2770 static void
2771 t4_set_desc(struct adapter *sc)
2772 {
2773         char buf[128];
2774         struct adapter_params *p = &sc->params;
2775
2776         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2777             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2778             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2779
2780         device_set_desc_copy(sc->dev, buf);
2781 }
2782
2783 static void
2784 build_medialist(struct port_info *pi, struct ifmedia *media)
2785 {
2786         int data, m;
2787
2788         PORT_LOCK(pi);
2789
2790         ifmedia_removeall(media);
2791
2792         m = IFM_ETHER | IFM_FDX;
2793         data = (pi->port_type << 8) | pi->mod_type;
2794
2795         switch(pi->port_type) {
2796         case FW_PORT_TYPE_BT_XFI:
2797         case FW_PORT_TYPE_BT_XAUI:
2798                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2799                 /* fall through */
2800
2801         case FW_PORT_TYPE_BT_SGMII:
2802                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2803                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2804                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2805                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2806                 break;
2807
2808         case FW_PORT_TYPE_CX4:
2809                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2810                 ifmedia_set(media, m | IFM_10G_CX4);
2811                 break;
2812
2813         case FW_PORT_TYPE_QSFP_10G:
2814         case FW_PORT_TYPE_SFP:
2815         case FW_PORT_TYPE_FIBER_XFI:
2816         case FW_PORT_TYPE_FIBER_XAUI:
2817                 switch (pi->mod_type) {
2818
2819                 case FW_PORT_MOD_TYPE_LR:
2820                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2821                         ifmedia_set(media, m | IFM_10G_LR);
2822                         break;
2823
2824                 case FW_PORT_MOD_TYPE_SR:
2825                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2826                         ifmedia_set(media, m | IFM_10G_SR);
2827                         break;
2828
2829                 case FW_PORT_MOD_TYPE_LRM:
2830                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2831                         ifmedia_set(media, m | IFM_10G_LRM);
2832                         break;
2833
2834                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2835                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2836                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2837                         ifmedia_set(media, m | IFM_10G_TWINAX);
2838                         break;
2839
2840                 case FW_PORT_MOD_TYPE_NONE:
2841                         m &= ~IFM_FDX;
2842                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2843                         ifmedia_set(media, m | IFM_NONE);
2844                         break;
2845
2846                 case FW_PORT_MOD_TYPE_NA:
2847                 case FW_PORT_MOD_TYPE_ER:
2848                 default:
2849                         device_printf(pi->dev,
2850                             "unknown port_type (%d), mod_type (%d)\n",
2851                             pi->port_type, pi->mod_type);
2852                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2853                         ifmedia_set(media, m | IFM_UNKNOWN);
2854                         break;
2855                 }
2856                 break;
2857
2858         case FW_PORT_TYPE_QSFP:
2859                 switch (pi->mod_type) {
2860
2861                 case FW_PORT_MOD_TYPE_LR:
2862                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2863                         ifmedia_set(media, m | IFM_40G_LR4);
2864                         break;
2865
2866                 case FW_PORT_MOD_TYPE_SR:
2867                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2868                         ifmedia_set(media, m | IFM_40G_SR4);
2869                         break;
2870
2871                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2872                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2873                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2874                         ifmedia_set(media, m | IFM_40G_CR4);
2875                         break;
2876
2877                 case FW_PORT_MOD_TYPE_NONE:
2878                         m &= ~IFM_FDX;
2879                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2880                         ifmedia_set(media, m | IFM_NONE);
2881                         break;
2882
2883                 default:
2884                         device_printf(pi->dev,
2885                             "unknown port_type (%d), mod_type (%d)\n",
2886                             pi->port_type, pi->mod_type);
2887                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2888                         ifmedia_set(media, m | IFM_UNKNOWN);
2889                         break;
2890                 }
2891                 break;
2892
2893         default:
2894                 device_printf(pi->dev,
2895                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2896                     pi->mod_type);
2897                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2898                 ifmedia_set(media, m | IFM_UNKNOWN);
2899                 break;
2900         }
2901
2902         PORT_UNLOCK(pi);
2903 }
2904
2905 #define FW_MAC_EXACT_CHUNK      7
2906
2907 /*
2908  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2909  * indicates which parameters should be programmed (the rest are left alone).
2910  */
2911 int
2912 update_mac_settings(struct ifnet *ifp, int flags)
2913 {
2914         int rc = 0;
2915         struct port_info *pi = ifp->if_softc;
2916         struct adapter *sc = pi->adapter;
2917         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2918         uint16_t viid = 0xffff;
2919         int16_t *xact_addr_filt = NULL;
2920
2921         ASSERT_SYNCHRONIZED_OP(sc);
2922         KASSERT(flags, ("%s: not told what to update.", __func__));
2923
2924         if (ifp == pi->ifp) {
2925                 viid = pi->viid;
2926                 xact_addr_filt = &pi->xact_addr_filt;
2927         }
2928 #ifdef DEV_NETMAP
2929         else if (ifp == pi->nm_ifp) {
2930                 viid = pi->nm_viid;
2931                 xact_addr_filt = &pi->nm_xact_addr_filt;
2932         }
2933 #endif
2934         if (flags & XGMAC_MTU)
2935                 mtu = ifp->if_mtu;
2936
2937         if (flags & XGMAC_PROMISC)
2938                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2939
2940         if (flags & XGMAC_ALLMULTI)
2941                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2942
2943         if (flags & XGMAC_VLANEX)
2944                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2945
2946         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2947                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2948                     1, vlanex, false);
2949                 if (rc) {
2950                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2951                             rc);
2952                         return (rc);
2953                 }
2954         }
2955
2956         if (flags & XGMAC_UCADDR) {
2957                 uint8_t ucaddr[ETHER_ADDR_LEN];
2958
2959                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2960                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2961                     true, true);
2962                 if (rc < 0) {
2963                         rc = -rc;
2964                         if_printf(ifp, "change_mac failed: %d\n", rc);
2965                         return (rc);
2966                 } else {
2967                         *xact_addr_filt = rc;
2968                         rc = 0;
2969                 }
2970         }
2971
2972         if (flags & XGMAC_MCADDRS) {
2973                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2974                 int del = 1;
2975                 uint64_t hash = 0;
2976                 struct ifmultiaddr *ifma;
2977                 int i = 0, j;
2978
2979                 if_maddr_rlock(ifp);
2980                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2981                         if (ifma->ifma_addr->sa_family != AF_LINK)
2982                                 continue;
2983                         mcaddr[i] =
2984                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2985                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
2986                         i++;
2987
2988                         if (i == FW_MAC_EXACT_CHUNK) {
2989                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2990                                     i, mcaddr, NULL, &hash, 0);
2991                                 if (rc < 0) {
2992                                         rc = -rc;
2993                                         for (j = 0; j < i; j++) {
2994                                                 if_printf(ifp,
2995                                                     "failed to add mc address"
2996                                                     " %02x:%02x:%02x:"
2997                                                     "%02x:%02x:%02x rc=%d\n",
2998                                                     mcaddr[j][0], mcaddr[j][1],
2999                                                     mcaddr[j][2], mcaddr[j][3],
3000                                                     mcaddr[j][4], mcaddr[j][5],
3001                                                     rc);
3002                                         }
3003                                         goto mcfail;
3004                                 }
3005                                 del = 0;
3006                                 i = 0;
3007                         }
3008                 }
3009                 if (i > 0) {
3010                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3011                             mcaddr, NULL, &hash, 0);
3012                         if (rc < 0) {
3013                                 rc = -rc;
3014                                 for (j = 0; j < i; j++) {
3015                                         if_printf(ifp,
3016                                             "failed to add mc address"
3017                                             " %02x:%02x:%02x:"
3018                                             "%02x:%02x:%02x rc=%d\n",
3019                                             mcaddr[j][0], mcaddr[j][1],
3020                                             mcaddr[j][2], mcaddr[j][3],
3021                                             mcaddr[j][4], mcaddr[j][5],
3022                                             rc);
3023                                 }
3024                                 goto mcfail;
3025                         }
3026                 }
3027
3028                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3029                 if (rc != 0)
3030                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3031 mcfail:
3032                 if_maddr_runlock(ifp);
3033         }
3034
3035         return (rc);
3036 }
3037
3038 /*
3039  * {begin|end}_synchronized_op must be called from the same thread.
3040  */
3041 int
3042 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3043     char *wmesg)
3044 {
3045         int rc, pri;
3046
3047 #ifdef WITNESS
3048         /* the caller thinks it's ok to sleep, but is it really? */
3049         if (flags & SLEEP_OK)
3050                 pause("t4slptst", 1);
3051 #endif
3052
3053         if (INTR_OK)
3054                 pri = PCATCH;
3055         else
3056                 pri = 0;
3057
3058         ADAPTER_LOCK(sc);
3059         for (;;) {
3060
3061                 if (pi && IS_DOOMED(pi)) {
3062                         rc = ENXIO;
3063                         goto done;
3064                 }
3065
3066                 if (!IS_BUSY(sc)) {
3067                         rc = 0;
3068                         break;
3069                 }
3070
3071                 if (!(flags & SLEEP_OK)) {
3072                         rc = EBUSY;
3073                         goto done;
3074                 }
3075
3076                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3077                         rc = EINTR;
3078                         goto done;
3079                 }
3080         }
3081
3082         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3083         SET_BUSY(sc);
3084 #ifdef INVARIANTS
3085         sc->last_op = wmesg;
3086         sc->last_op_thr = curthread;
3087 #endif
3088
3089 done:
3090         if (!(flags & HOLD_LOCK) || rc)
3091                 ADAPTER_UNLOCK(sc);
3092
3093         return (rc);
3094 }
3095
3096 /*
3097  * {begin|end}_synchronized_op must be called from the same thread.
3098  */
3099 void
3100 end_synchronized_op(struct adapter *sc, int flags)
3101 {
3102
3103         if (flags & LOCK_HELD)
3104                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3105         else
3106                 ADAPTER_LOCK(sc);
3107
3108         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3109         CLR_BUSY(sc);
3110         wakeup(&sc->flags);
3111         ADAPTER_UNLOCK(sc);
3112 }
3113
3114 static int
3115 cxgbe_init_synchronized(struct port_info *pi)
3116 {
3117         struct adapter *sc = pi->adapter;
3118         struct ifnet *ifp = pi->ifp;
3119         int rc = 0, i;
3120         struct sge_txq *txq;
3121
3122         ASSERT_SYNCHRONIZED_OP(sc);
3123
3124         if (isset(&sc->open_device_map, pi->port_id)) {
3125                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3126                     ("mismatch between open_device_map and if_drv_flags"));
3127                 return (0);     /* already running */
3128         }
3129
3130         if (!(sc->flags & FULL_INIT_DONE) &&
3131             ((rc = adapter_full_init(sc)) != 0))
3132                 return (rc);    /* error message displayed already */
3133
3134         if (!(pi->flags & PORT_INIT_DONE) &&
3135             ((rc = port_full_init(pi)) != 0))
3136                 return (rc); /* error message displayed already */
3137
3138         rc = update_mac_settings(ifp, XGMAC_ALL);
3139         if (rc)
3140                 goto done;      /* error message displayed already */
3141
3142         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3143         if (rc != 0) {
3144                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3145                 goto done;
3146         }
3147
3148         /*
3149          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3150          * if this changes.
3151          */
3152
3153         for_each_txq(pi, i, txq) {
3154                 TXQ_LOCK(txq);
3155                 txq->eq.flags |= EQ_ENABLED;
3156                 TXQ_UNLOCK(txq);
3157         }
3158
3159         /*
3160          * The first iq of the first port to come up is used for tracing.
3161          */
3162         if (sc->traceq < 0) {
3163                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3164                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3165                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3166                     V_QUEUENUMBER(sc->traceq));
3167                 pi->flags |= HAS_TRACEQ;
3168         }
3169
3170         /* all ok */
3171         setbit(&sc->open_device_map, pi->port_id);
3172         PORT_LOCK(pi);
3173         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3174         PORT_UNLOCK(pi);
3175
3176         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3177 done:
3178         if (rc != 0)
3179                 cxgbe_uninit_synchronized(pi);
3180
3181         return (rc);
3182 }
3183
3184 /*
3185  * Idempotent.
3186  */
3187 static int
3188 cxgbe_uninit_synchronized(struct port_info *pi)
3189 {
3190         struct adapter *sc = pi->adapter;
3191         struct ifnet *ifp = pi->ifp;
3192         int rc, i;
3193         struct sge_txq *txq;
3194
3195         ASSERT_SYNCHRONIZED_OP(sc);
3196
3197         /*
3198          * Disable the VI so that all its data in either direction is discarded
3199          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3200          * tick) intact as the TP can deliver negative advice or data that it's
3201          * holding in its RAM (for an offloaded connection) even after the VI is
3202          * disabled.
3203          */
3204         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3205         if (rc) {
3206                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3207                 return (rc);
3208         }
3209
3210         for_each_txq(pi, i, txq) {
3211                 TXQ_LOCK(txq);
3212                 txq->eq.flags &= ~EQ_ENABLED;
3213                 TXQ_UNLOCK(txq);
3214         }
3215
3216         clrbit(&sc->open_device_map, pi->port_id);
3217         PORT_LOCK(pi);
3218         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3219         PORT_UNLOCK(pi);
3220
3221         pi->link_cfg.link_ok = 0;
3222         pi->link_cfg.speed = 0;
3223         pi->linkdnrc = -1;
3224         t4_os_link_changed(sc, pi->port_id, 0, -1);
3225
3226         return (0);
3227 }
3228
3229 /*
3230  * It is ok for this function to fail midway and return right away.  t4_detach
3231  * will walk the entire sc->irq list and clean up whatever is valid.
3232  */
3233 static int
3234 setup_intr_handlers(struct adapter *sc)
3235 {
3236         int rc, rid, p, q;
3237         char s[8];
3238         struct irq *irq;
3239         struct port_info *pi;
3240         struct sge_rxq *rxq;
3241 #ifdef TCP_OFFLOAD
3242         struct sge_ofld_rxq *ofld_rxq;
3243 #endif
3244 #ifdef DEV_NETMAP
3245         struct sge_nm_rxq *nm_rxq;
3246 #endif
3247
3248         /*
3249          * Setup interrupts.
3250          */
3251         irq = &sc->irq[0];
3252         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3253         if (sc->intr_count == 1)
3254                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3255
3256         /* Multiple interrupts. */
3257         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3258             ("%s: too few intr.", __func__));
3259
3260         /* The first one is always error intr */
3261         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3262         if (rc != 0)
3263                 return (rc);
3264         irq++;
3265         rid++;
3266
3267         /* The second one is always the firmware event queue */
3268         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3269         if (rc != 0)
3270                 return (rc);
3271         irq++;
3272         rid++;
3273
3274         for_each_port(sc, p) {
3275                 pi = sc->port[p];
3276
3277                 if (pi->flags & INTR_RXQ) {
3278                         for_each_rxq(pi, q, rxq) {
3279                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3280                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3281                                     s);
3282                                 if (rc != 0)
3283                                         return (rc);
3284                                 irq++;
3285                                 rid++;
3286                         }
3287                 }
3288 #ifdef TCP_OFFLOAD
3289                 if (pi->flags & INTR_OFLD_RXQ) {
3290                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3291                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3292                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3293                                     ofld_rxq, s);
3294                                 if (rc != 0)
3295                                         return (rc);
3296                                 irq++;
3297                                 rid++;
3298                         }
3299                 }
3300 #endif
3301 #ifdef DEV_NETMAP
3302                 if (pi->flags & INTR_NM_RXQ) {
3303                         for_each_nm_rxq(pi, q, nm_rxq) {
3304                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3305                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3306                                     nm_rxq, s);
3307                                 if (rc != 0)
3308                                         return (rc);
3309                                 irq++;
3310                                 rid++;
3311                         }
3312                 }
3313 #endif
3314         }
3315         MPASS(irq == &sc->irq[sc->intr_count]);
3316
3317         return (0);
3318 }
3319
3320 int
3321 adapter_full_init(struct adapter *sc)
3322 {
3323         int rc, i;
3324
3325         ASSERT_SYNCHRONIZED_OP(sc);
3326         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3327         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3328             ("%s: FULL_INIT_DONE already", __func__));
3329
3330         /*
3331          * queues that belong to the adapter (not any particular port).
3332          */
3333         rc = t4_setup_adapter_queues(sc);
3334         if (rc != 0)
3335                 goto done;
3336
3337         for (i = 0; i < nitems(sc->tq); i++) {
3338                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3339                     taskqueue_thread_enqueue, &sc->tq[i]);
3340                 if (sc->tq[i] == NULL) {
3341                         device_printf(sc->dev,
3342                             "failed to allocate task queue %d\n", i);
3343                         rc = ENOMEM;
3344                         goto done;
3345                 }
3346                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3347                     device_get_nameunit(sc->dev), i);
3348         }
3349
3350         t4_intr_enable(sc);
3351         sc->flags |= FULL_INIT_DONE;
3352 done:
3353         if (rc != 0)
3354                 adapter_full_uninit(sc);
3355
3356         return (rc);
3357 }
3358
3359 int
3360 adapter_full_uninit(struct adapter *sc)
3361 {
3362         int i;
3363
3364         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3365
3366         t4_teardown_adapter_queues(sc);
3367
3368         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3369                 taskqueue_free(sc->tq[i]);
3370                 sc->tq[i] = NULL;
3371         }
3372
3373         sc->flags &= ~FULL_INIT_DONE;
3374
3375         return (0);
3376 }
3377
3378 int
3379 port_full_init(struct port_info *pi)
3380 {
3381         struct adapter *sc = pi->adapter;
3382         struct ifnet *ifp = pi->ifp;
3383         uint16_t *rss;
3384         struct sge_rxq *rxq;
3385         int rc, i, j;
3386
3387         ASSERT_SYNCHRONIZED_OP(sc);
3388         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3389             ("%s: PORT_INIT_DONE already", __func__));
3390
3391         sysctl_ctx_init(&pi->ctx);
3392         pi->flags |= PORT_SYSCTL_CTX;
3393
3394         /*
3395          * Allocate tx/rx/fl queues for this port.
3396          */
3397         rc = t4_setup_port_queues(pi);
3398         if (rc != 0)
3399                 goto done;      /* error message displayed already */
3400
3401         /*
3402          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3403          */
3404         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3405         for (i = 0; i < pi->rss_size;) {
3406                 for_each_rxq(pi, j, rxq) {
3407                         rss[i++] = rxq->iq.abs_id;
3408                         if (i == pi->rss_size)
3409                                 break;
3410                 }
3411         }
3412
3413         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3414             pi->rss_size);
3415         if (rc != 0) {
3416                 if_printf(ifp, "rss_config failed: %d\n", rc);
3417                 goto done;
3418         }
3419
3420         pi->rss = rss;
3421         pi->flags |= PORT_INIT_DONE;
3422 done:
3423         if (rc != 0)
3424                 port_full_uninit(pi);
3425
3426         return (rc);
3427 }
3428
3429 /*
3430  * Idempotent.
3431  */
3432 int
3433 port_full_uninit(struct port_info *pi)
3434 {
3435         struct adapter *sc = pi->adapter;
3436         int i;
3437         struct sge_rxq *rxq;
3438         struct sge_txq *txq;
3439 #ifdef TCP_OFFLOAD
3440         struct sge_ofld_rxq *ofld_rxq;
3441         struct sge_wrq *ofld_txq;
3442 #endif
3443
3444         if (pi->flags & PORT_INIT_DONE) {
3445
3446                 /* Need to quiesce queues.  */
3447
3448                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3449
3450                 for_each_txq(pi, i, txq) {
3451                         quiesce_txq(sc, txq);
3452                 }
3453
3454 #ifdef TCP_OFFLOAD
3455                 for_each_ofld_txq(pi, i, ofld_txq) {
3456                         quiesce_wrq(sc, ofld_txq);
3457                 }
3458 #endif
3459
3460                 for_each_rxq(pi, i, rxq) {
3461                         quiesce_iq(sc, &rxq->iq);
3462                         quiesce_fl(sc, &rxq->fl);
3463                 }
3464
3465 #ifdef TCP_OFFLOAD
3466                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3467                         quiesce_iq(sc, &ofld_rxq->iq);
3468                         quiesce_fl(sc, &ofld_rxq->fl);
3469                 }
3470 #endif
3471                 free(pi->rss, M_CXGBE);
3472         }
3473
3474         t4_teardown_port_queues(pi);
3475         pi->flags &= ~PORT_INIT_DONE;
3476
3477         return (0);
3478 }
3479
3480 static void
3481 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3482 {
3483         struct sge_eq *eq = &txq->eq;
3484         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3485
3486         (void) sc;      /* unused */
3487
3488 #ifdef INVARIANTS
3489         TXQ_LOCK(txq);
3490         MPASS((eq->flags & EQ_ENABLED) == 0);
3491         TXQ_UNLOCK(txq);
3492 #endif
3493
3494         /* Wait for the mp_ring to empty. */
3495         while (!mp_ring_is_idle(txq->r)) {
3496                 mp_ring_check_drainage(txq->r, 0);
3497                 pause("rquiesce", 1);
3498         }
3499
3500         /* Then wait for the hardware to finish. */
3501         while (spg->cidx != htobe16(eq->pidx))
3502                 pause("equiesce", 1);
3503
3504         /* Finally, wait for the driver to reclaim all descriptors. */
3505         while (eq->cidx != eq->pidx)
3506                 pause("dquiesce", 1);
3507 }
3508
3509 static void
3510 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3511 {
3512
3513         /* XXXTX */
3514 }
3515
3516 static void
3517 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3518 {
3519         (void) sc;      /* unused */
3520
3521         /* Synchronize with the interrupt handler */
3522         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3523                 pause("iqfree", 1);
3524 }
3525
3526 static void
3527 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3528 {
3529         mtx_lock(&sc->sfl_lock);
3530         FL_LOCK(fl);
3531         fl->flags |= FL_DOOMED;
3532         FL_UNLOCK(fl);
3533         mtx_unlock(&sc->sfl_lock);
3534
3535         callout_drain(&sc->sfl_callout);
3536         KASSERT((fl->flags & FL_STARVING) == 0,
3537             ("%s: still starving", __func__));
3538 }
3539
3540 static int
3541 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3542     driver_intr_t *handler, void *arg, char *name)
3543 {
3544         int rc;
3545
3546         irq->rid = rid;
3547         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3548             RF_SHAREABLE | RF_ACTIVE);
3549         if (irq->res == NULL) {
3550                 device_printf(sc->dev,
3551                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3552                 return (ENOMEM);
3553         }
3554
3555         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3556             NULL, handler, arg, &irq->tag);
3557         if (rc != 0) {
3558                 device_printf(sc->dev,
3559                     "failed to setup interrupt for rid %d, name %s: %d\n",
3560                     rid, name, rc);
3561         } else if (name)
3562                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3563
3564         return (rc);
3565 }
3566
3567 static int
3568 t4_free_irq(struct adapter *sc, struct irq *irq)
3569 {
3570         if (irq->tag)
3571                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3572         if (irq->res)
3573                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3574
3575         bzero(irq, sizeof(*irq));
3576
3577         return (0);
3578 }
3579
3580 static void
3581 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3582     unsigned int end)
3583 {
3584         uint32_t *p = (uint32_t *)(buf + start);
3585
3586         for ( ; start <= end; start += sizeof(uint32_t))
3587                 *p++ = t4_read_reg(sc, start);
3588 }
3589
3590 static void
3591 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3592 {
3593         int i, n;
3594         const unsigned int *reg_ranges;
3595         static const unsigned int t4_reg_ranges[] = {
3596                 0x1008, 0x1108,
3597                 0x1180, 0x11b4,
3598                 0x11fc, 0x123c,
3599                 0x1300, 0x173c,
3600                 0x1800, 0x18fc,
3601                 0x3000, 0x30d8,
3602                 0x30e0, 0x5924,
3603                 0x5960, 0x59d4,
3604                 0x5a00, 0x5af8,
3605                 0x6000, 0x6098,
3606                 0x6100, 0x6150,
3607                 0x6200, 0x6208,
3608                 0x6240, 0x6248,
3609                 0x6280, 0x6338,
3610                 0x6370, 0x638c,
3611                 0x6400, 0x643c,
3612                 0x6500, 0x6524,
3613                 0x6a00, 0x6a38,
3614                 0x6a60, 0x6a78,
3615                 0x6b00, 0x6b84,
3616                 0x6bf0, 0x6c84,
3617                 0x6cf0, 0x6d84,
3618                 0x6df0, 0x6e84,
3619                 0x6ef0, 0x6f84,
3620                 0x6ff0, 0x7084,
3621                 0x70f0, 0x7184,
3622                 0x71f0, 0x7284,
3623                 0x72f0, 0x7384,
3624                 0x73f0, 0x7450,
3625                 0x7500, 0x7530,
3626                 0x7600, 0x761c,
3627                 0x7680, 0x76cc,
3628                 0x7700, 0x7798,
3629                 0x77c0, 0x77fc,
3630                 0x7900, 0x79fc,
3631                 0x7b00, 0x7c38,
3632                 0x7d00, 0x7efc,
3633                 0x8dc0, 0x8e1c,
3634                 0x8e30, 0x8e78,
3635                 0x8ea0, 0x8f6c,
3636                 0x8fc0, 0x9074,
3637                 0x90fc, 0x90fc,
3638                 0x9400, 0x9458,
3639                 0x9600, 0x96bc,
3640                 0x9800, 0x9808,
3641                 0x9820, 0x983c,
3642                 0x9850, 0x9864,
3643                 0x9c00, 0x9c6c,
3644                 0x9c80, 0x9cec,
3645                 0x9d00, 0x9d6c,
3646                 0x9d80, 0x9dec,
3647                 0x9e00, 0x9e6c,
3648                 0x9e80, 0x9eec,
3649                 0x9f00, 0x9f6c,
3650                 0x9f80, 0x9fec,
3651                 0xd004, 0xd03c,
3652                 0xdfc0, 0xdfe0,
3653                 0xe000, 0xea7c,
3654                 0xf000, 0x11110,
3655                 0x11118, 0x11190,
3656                 0x19040, 0x1906c,
3657                 0x19078, 0x19080,
3658                 0x1908c, 0x19124,
3659                 0x19150, 0x191b0,
3660                 0x191d0, 0x191e8,
3661                 0x19238, 0x1924c,
3662                 0x193f8, 0x19474,
3663                 0x19490, 0x194f8,
3664                 0x19800, 0x19f30,
3665                 0x1a000, 0x1a06c,
3666                 0x1a0b0, 0x1a120,
3667                 0x1a128, 0x1a138,
3668                 0x1a190, 0x1a1c4,
3669                 0x1a1fc, 0x1a1fc,
3670                 0x1e040, 0x1e04c,
3671                 0x1e284, 0x1e28c,
3672                 0x1e2c0, 0x1e2c0,
3673                 0x1e2e0, 0x1e2e0,
3674                 0x1e300, 0x1e384,
3675                 0x1e3c0, 0x1e3c8,
3676                 0x1e440, 0x1e44c,
3677                 0x1e684, 0x1e68c,
3678                 0x1e6c0, 0x1e6c0,
3679                 0x1e6e0, 0x1e6e0,
3680                 0x1e700, 0x1e784,
3681                 0x1e7c0, 0x1e7c8,
3682                 0x1e840, 0x1e84c,
3683                 0x1ea84, 0x1ea8c,
3684                 0x1eac0, 0x1eac0,
3685                 0x1eae0, 0x1eae0,
3686                 0x1eb00, 0x1eb84,
3687                 0x1ebc0, 0x1ebc8,
3688                 0x1ec40, 0x1ec4c,
3689                 0x1ee84, 0x1ee8c,
3690                 0x1eec0, 0x1eec0,
3691                 0x1eee0, 0x1eee0,
3692                 0x1ef00, 0x1ef84,
3693                 0x1efc0, 0x1efc8,
3694                 0x1f040, 0x1f04c,
3695                 0x1f284, 0x1f28c,
3696                 0x1f2c0, 0x1f2c0,
3697                 0x1f2e0, 0x1f2e0,
3698                 0x1f300, 0x1f384,
3699                 0x1f3c0, 0x1f3c8,
3700                 0x1f440, 0x1f44c,
3701                 0x1f684, 0x1f68c,
3702                 0x1f6c0, 0x1f6c0,
3703                 0x1f6e0, 0x1f6e0,
3704                 0x1f700, 0x1f784,
3705                 0x1f7c0, 0x1f7c8,
3706                 0x1f840, 0x1f84c,
3707                 0x1fa84, 0x1fa8c,
3708                 0x1fac0, 0x1fac0,
3709                 0x1fae0, 0x1fae0,
3710                 0x1fb00, 0x1fb84,
3711                 0x1fbc0, 0x1fbc8,
3712                 0x1fc40, 0x1fc4c,
3713                 0x1fe84, 0x1fe8c,
3714                 0x1fec0, 0x1fec0,
3715                 0x1fee0, 0x1fee0,
3716                 0x1ff00, 0x1ff84,
3717                 0x1ffc0, 0x1ffc8,
3718                 0x20000, 0x2002c,
3719                 0x20100, 0x2013c,
3720                 0x20190, 0x201c8,
3721                 0x20200, 0x20318,
3722                 0x20400, 0x20528,
3723                 0x20540, 0x20614,
3724                 0x21000, 0x21040,
3725                 0x2104c, 0x21060,
3726                 0x210c0, 0x210ec,
3727                 0x21200, 0x21268,
3728                 0x21270, 0x21284,
3729                 0x212fc, 0x21388,
3730                 0x21400, 0x21404,
3731                 0x21500, 0x21518,
3732                 0x2152c, 0x2153c,
3733                 0x21550, 0x21554,
3734                 0x21600, 0x21600,
3735                 0x21608, 0x21628,
3736                 0x21630, 0x2163c,
3737                 0x21700, 0x2171c,
3738                 0x21780, 0x2178c,
3739                 0x21800, 0x21c38,
3740                 0x21c80, 0x21d7c,
3741                 0x21e00, 0x21e04,
3742                 0x22000, 0x2202c,
3743                 0x22100, 0x2213c,
3744                 0x22190, 0x221c8,
3745                 0x22200, 0x22318,
3746                 0x22400, 0x22528,
3747                 0x22540, 0x22614,
3748                 0x23000, 0x23040,
3749                 0x2304c, 0x23060,
3750                 0x230c0, 0x230ec,
3751                 0x23200, 0x23268,
3752                 0x23270, 0x23284,
3753                 0x232fc, 0x23388,
3754                 0x23400, 0x23404,
3755                 0x23500, 0x23518,
3756                 0x2352c, 0x2353c,
3757                 0x23550, 0x23554,
3758                 0x23600, 0x23600,
3759                 0x23608, 0x23628,
3760                 0x23630, 0x2363c,
3761                 0x23700, 0x2371c,
3762                 0x23780, 0x2378c,
3763                 0x23800, 0x23c38,
3764                 0x23c80, 0x23d7c,
3765                 0x23e00, 0x23e04,
3766                 0x24000, 0x2402c,
3767                 0x24100, 0x2413c,
3768                 0x24190, 0x241c8,
3769                 0x24200, 0x24318,
3770                 0x24400, 0x24528,
3771                 0x24540, 0x24614,
3772                 0x25000, 0x25040,
3773                 0x2504c, 0x25060,
3774                 0x250c0, 0x250ec,
3775                 0x25200, 0x25268,
3776                 0x25270, 0x25284,
3777                 0x252fc, 0x25388,
3778                 0x25400, 0x25404,
3779                 0x25500, 0x25518,
3780                 0x2552c, 0x2553c,
3781                 0x25550, 0x25554,
3782                 0x25600, 0x25600,
3783                 0x25608, 0x25628,
3784                 0x25630, 0x2563c,
3785                 0x25700, 0x2571c,
3786                 0x25780, 0x2578c,
3787                 0x25800, 0x25c38,
3788                 0x25c80, 0x25d7c,
3789                 0x25e00, 0x25e04,
3790                 0x26000, 0x2602c,
3791                 0x26100, 0x2613c,
3792                 0x26190, 0x261c8,
3793                 0x26200, 0x26318,
3794                 0x26400, 0x26528,
3795                 0x26540, 0x26614,
3796                 0x27000, 0x27040,
3797                 0x2704c, 0x27060,
3798                 0x270c0, 0x270ec,
3799                 0x27200, 0x27268,
3800                 0x27270, 0x27284,
3801                 0x272fc, 0x27388,
3802                 0x27400, 0x27404,
3803                 0x27500, 0x27518,
3804                 0x2752c, 0x2753c,
3805                 0x27550, 0x27554,
3806                 0x27600, 0x27600,
3807                 0x27608, 0x27628,
3808                 0x27630, 0x2763c,
3809                 0x27700, 0x2771c,
3810                 0x27780, 0x2778c,
3811                 0x27800, 0x27c38,
3812                 0x27c80, 0x27d7c,
3813                 0x27e00, 0x27e04
3814         };
3815         static const unsigned int t5_reg_ranges[] = {
3816                 0x1008, 0x1148,
3817                 0x1180, 0x11b4,
3818                 0x11fc, 0x123c,
3819                 0x1280, 0x173c,
3820                 0x1800, 0x18fc,
3821                 0x3000, 0x3028,
3822                 0x3060, 0x30d8,
3823                 0x30e0, 0x30fc,
3824                 0x3140, 0x357c,
3825                 0x35a8, 0x35cc,
3826                 0x35ec, 0x35ec,
3827                 0x3600, 0x5624,
3828                 0x56cc, 0x575c,
3829                 0x580c, 0x5814,
3830                 0x5890, 0x58bc,
3831                 0x5940, 0x59dc,
3832                 0x59fc, 0x5a18,
3833                 0x5a60, 0x5a9c,
3834                 0x5b94, 0x5bfc,
3835                 0x6000, 0x6040,
3836                 0x6058, 0x614c,
3837                 0x7700, 0x7798,
3838                 0x77c0, 0x78fc,
3839                 0x7b00, 0x7c54,
3840                 0x7d00, 0x7efc,
3841                 0x8dc0, 0x8de0,
3842                 0x8df8, 0x8e84,
3843                 0x8ea0, 0x8f84,
3844                 0x8fc0, 0x90f8,
3845                 0x9400, 0x9470,
3846                 0x9600, 0x96f4,
3847                 0x9800, 0x9808,
3848                 0x9820, 0x983c,
3849                 0x9850, 0x9864,
3850                 0x9c00, 0x9c6c,
3851                 0x9c80, 0x9cec,
3852                 0x9d00, 0x9d6c,
3853                 0x9d80, 0x9dec,
3854                 0x9e00, 0x9e6c,
3855                 0x9e80, 0x9eec,
3856                 0x9f00, 0x9f6c,
3857                 0x9f80, 0xa020,
3858                 0xd004, 0xd03c,
3859                 0xdfc0, 0xdfe0,
3860                 0xe000, 0x11088,
3861                 0x1109c, 0x11110,
3862                 0x11118, 0x1117c,
3863                 0x11190, 0x11204,
3864                 0x19040, 0x1906c,
3865                 0x19078, 0x19080,
3866                 0x1908c, 0x19124,
3867                 0x19150, 0x191b0,
3868                 0x191d0, 0x191e8,
3869                 0x19238, 0x19290,
3870                 0x193f8, 0x19474,
3871                 0x19490, 0x194cc,
3872                 0x194f0, 0x194f8,
3873                 0x19c00, 0x19c60,
3874                 0x19c94, 0x19e10,
3875                 0x19e50, 0x19f34,
3876                 0x19f40, 0x19f50,
3877                 0x19f90, 0x19fe4,
3878                 0x1a000, 0x1a06c,
3879                 0x1a0b0, 0x1a120,
3880                 0x1a128, 0x1a138,
3881                 0x1a190, 0x1a1c4,
3882                 0x1a1fc, 0x1a1fc,
3883                 0x1e008, 0x1e00c,
3884                 0x1e040, 0x1e04c,
3885                 0x1e284, 0x1e290,
3886                 0x1e2c0, 0x1e2c0,
3887                 0x1e2e0, 0x1e2e0,
3888                 0x1e300, 0x1e384,
3889                 0x1e3c0, 0x1e3c8,
3890                 0x1e408, 0x1e40c,
3891                 0x1e440, 0x1e44c,
3892                 0x1e684, 0x1e690,
3893                 0x1e6c0, 0x1e6c0,
3894                 0x1e6e0, 0x1e6e0,
3895                 0x1e700, 0x1e784,
3896                 0x1e7c0, 0x1e7c8,
3897                 0x1e808, 0x1e80c,
3898                 0x1e840, 0x1e84c,
3899                 0x1ea84, 0x1ea90,
3900                 0x1eac0, 0x1eac0,
3901                 0x1eae0, 0x1eae0,
3902                 0x1eb00, 0x1eb84,
3903                 0x1ebc0, 0x1ebc8,
3904                 0x1ec08, 0x1ec0c,
3905                 0x1ec40, 0x1ec4c,
3906                 0x1ee84, 0x1ee90,
3907                 0x1eec0, 0x1eec0,
3908                 0x1eee0, 0x1eee0,
3909                 0x1ef00, 0x1ef84,
3910                 0x1efc0, 0x1efc8,
3911                 0x1f008, 0x1f00c,
3912                 0x1f040, 0x1f04c,
3913                 0x1f284, 0x1f290,
3914                 0x1f2c0, 0x1f2c0,
3915                 0x1f2e0, 0x1f2e0,
3916                 0x1f300, 0x1f384,
3917                 0x1f3c0, 0x1f3c8,
3918                 0x1f408, 0x1f40c,
3919                 0x1f440, 0x1f44c,
3920                 0x1f684, 0x1f690,
3921                 0x1f6c0, 0x1f6c0,
3922                 0x1f6e0, 0x1f6e0,
3923                 0x1f700, 0x1f784,
3924                 0x1f7c0, 0x1f7c8,
3925                 0x1f808, 0x1f80c,
3926                 0x1f840, 0x1f84c,
3927                 0x1fa84, 0x1fa90,
3928                 0x1fac0, 0x1fac0,
3929                 0x1fae0, 0x1fae0,
3930                 0x1fb00, 0x1fb84,
3931                 0x1fbc0, 0x1fbc8,
3932                 0x1fc08, 0x1fc0c,
3933                 0x1fc40, 0x1fc4c,
3934                 0x1fe84, 0x1fe90,
3935                 0x1fec0, 0x1fec0,
3936                 0x1fee0, 0x1fee0,
3937                 0x1ff00, 0x1ff84,
3938                 0x1ffc0, 0x1ffc8,
3939                 0x30000, 0x30030,
3940                 0x30100, 0x30144,
3941                 0x30190, 0x301d0,
3942                 0x30200, 0x30318,
3943                 0x30400, 0x3052c,
3944                 0x30540, 0x3061c,
3945                 0x30800, 0x30834,
3946                 0x308c0, 0x30908,
3947                 0x30910, 0x309ac,
3948                 0x30a00, 0x30a2c,
3949                 0x30a44, 0x30a50,
3950                 0x30a74, 0x30c24,
3951                 0x30d00, 0x30d00,
3952                 0x30d08, 0x30d14,
3953                 0x30d1c, 0x30d20,
3954                 0x30d3c, 0x30d50,
3955                 0x31200, 0x3120c,
3956                 0x31220, 0x31220,
3957                 0x31240, 0x31240,
3958                 0x31600, 0x3160c,
3959                 0x31a00, 0x31a1c,
3960                 0x31e00, 0x31e20,
3961                 0x31e38, 0x31e3c,
3962                 0x31e80, 0x31e80,
3963                 0x31e88, 0x31ea8,
3964                 0x31eb0, 0x31eb4,
3965                 0x31ec8, 0x31ed4,
3966                 0x31fb8, 0x32004,
3967                 0x32200, 0x32200,
3968                 0x32208, 0x32240,
3969                 0x32248, 0x32280,
3970                 0x32288, 0x322c0,
3971                 0x322c8, 0x322fc,
3972                 0x32600, 0x32630,
3973                 0x32a00, 0x32abc,
3974                 0x32b00, 0x32b70,
3975                 0x33000, 0x33048,
3976                 0x33060, 0x3309c,
3977                 0x330f0, 0x33148,
3978                 0x33160, 0x3319c,
3979                 0x331f0, 0x332e4,
3980                 0x332f8, 0x333e4,
3981                 0x333f8, 0x33448,
3982                 0x33460, 0x3349c,
3983                 0x334f0, 0x33548,
3984                 0x33560, 0x3359c,
3985                 0x335f0, 0x336e4,
3986                 0x336f8, 0x337e4,
3987                 0x337f8, 0x337fc,
3988                 0x33814, 0x33814,
3989                 0x3382c, 0x3382c,
3990                 0x33880, 0x3388c,
3991                 0x338e8, 0x338ec,
3992                 0x33900, 0x33948,
3993                 0x33960, 0x3399c,
3994                 0x339f0, 0x33ae4,
3995                 0x33af8, 0x33b10,
3996                 0x33b28, 0x33b28,
3997                 0x33b3c, 0x33b50,
3998                 0x33bf0, 0x33c10,
3999                 0x33c28, 0x33c28,
4000                 0x33c3c, 0x33c50,
4001                 0x33cf0, 0x33cfc,
4002                 0x34000, 0x34030,
4003                 0x34100, 0x34144,
4004                 0x34190, 0x341d0,
4005                 0x34200, 0x34318,
4006                 0x34400, 0x3452c,
4007                 0x34540, 0x3461c,
4008                 0x34800, 0x34834,
4009                 0x348c0, 0x34908,
4010                 0x34910, 0x349ac,
4011                 0x34a00, 0x34a2c,
4012                 0x34a44, 0x34a50,
4013                 0x34a74, 0x34c24,
4014                 0x34d00, 0x34d00,
4015                 0x34d08, 0x34d14,
4016                 0x34d1c, 0x34d20,
4017                 0x34d3c, 0x34d50,
4018                 0x35200, 0x3520c,
4019                 0x35220, 0x35220,
4020                 0x35240, 0x35240,
4021                 0x35600, 0x3560c,
4022                 0x35a00, 0x35a1c,
4023                 0x35e00, 0x35e20,
4024                 0x35e38, 0x35e3c,
4025                 0x35e80, 0x35e80,
4026                 0x35e88, 0x35ea8,
4027                 0x35eb0, 0x35eb4,
4028                 0x35ec8, 0x35ed4,
4029                 0x35fb8, 0x36004,
4030                 0x36200, 0x36200,
4031                 0x36208, 0x36240,
4032                 0x36248, 0x36280,
4033                 0x36288, 0x362c0,
4034                 0x362c8, 0x362fc,
4035                 0x36600, 0x36630,
4036                 0x36a00, 0x36abc,
4037                 0x36b00, 0x36b70,
4038                 0x37000, 0x37048,
4039                 0x37060, 0x3709c,
4040                 0x370f0, 0x37148,
4041                 0x37160, 0x3719c,
4042                 0x371f0, 0x372e4,
4043                 0x372f8, 0x373e4,
4044                 0x373f8, 0x37448,
4045                 0x37460, 0x3749c,
4046                 0x374f0, 0x37548,
4047                 0x37560, 0x3759c,
4048                 0x375f0, 0x376e4,
4049                 0x376f8, 0x377e4,
4050                 0x377f8, 0x377fc,
4051                 0x37814, 0x37814,
4052                 0x3782c, 0x3782c,
4053                 0x37880, 0x3788c,
4054                 0x378e8, 0x378ec,
4055                 0x37900, 0x37948,
4056                 0x37960, 0x3799c,
4057                 0x379f0, 0x37ae4,
4058                 0x37af8, 0x37b10,
4059                 0x37b28, 0x37b28,
4060                 0x37b3c, 0x37b50,
4061                 0x37bf0, 0x37c10,
4062                 0x37c28, 0x37c28,
4063                 0x37c3c, 0x37c50,
4064                 0x37cf0, 0x37cfc,
4065                 0x38000, 0x38030,
4066                 0x38100, 0x38144,
4067                 0x38190, 0x381d0,
4068                 0x38200, 0x38318,
4069                 0x38400, 0x3852c,
4070                 0x38540, 0x3861c,
4071                 0x38800, 0x38834,
4072                 0x388c0, 0x38908,
4073                 0x38910, 0x389ac,
4074                 0x38a00, 0x38a2c,
4075                 0x38a44, 0x38a50,
4076                 0x38a74, 0x38c24,
4077                 0x38d00, 0x38d00,
4078                 0x38d08, 0x38d14,
4079                 0x38d1c, 0x38d20,
4080                 0x38d3c, 0x38d50,
4081                 0x39200, 0x3920c,
4082                 0x39220, 0x39220,
4083                 0x39240, 0x39240,
4084                 0x39600, 0x3960c,
4085                 0x39a00, 0x39a1c,
4086                 0x39e00, 0x39e20,
4087                 0x39e38, 0x39e3c,
4088                 0x39e80, 0x39e80,
4089                 0x39e88, 0x39ea8,
4090                 0x39eb0, 0x39eb4,
4091                 0x39ec8, 0x39ed4,
4092                 0x39fb8, 0x3a004,
4093                 0x3a200, 0x3a200,
4094                 0x3a208, 0x3a240,
4095                 0x3a248, 0x3a280,
4096                 0x3a288, 0x3a2c0,
4097                 0x3a2c8, 0x3a2fc,
4098                 0x3a600, 0x3a630,
4099                 0x3aa00, 0x3aabc,
4100                 0x3ab00, 0x3ab70,
4101                 0x3b000, 0x3b048,
4102                 0x3b060, 0x3b09c,
4103                 0x3b0f0, 0x3b148,
4104                 0x3b160, 0x3b19c,
4105                 0x3b1f0, 0x3b2e4,
4106                 0x3b2f8, 0x3b3e4,
4107                 0x3b3f8, 0x3b448,
4108                 0x3b460, 0x3b49c,
4109                 0x3b4f0, 0x3b548,
4110                 0x3b560, 0x3b59c,
4111                 0x3b5f0, 0x3b6e4,
4112                 0x3b6f8, 0x3b7e4,
4113                 0x3b7f8, 0x3b7fc,
4114                 0x3b814, 0x3b814,
4115                 0x3b82c, 0x3b82c,
4116                 0x3b880, 0x3b88c,
4117                 0x3b8e8, 0x3b8ec,
4118                 0x3b900, 0x3b948,
4119                 0x3b960, 0x3b99c,
4120                 0x3b9f0, 0x3bae4,
4121                 0x3baf8, 0x3bb10,
4122                 0x3bb28, 0x3bb28,
4123                 0x3bb3c, 0x3bb50,
4124                 0x3bbf0, 0x3bc10,
4125                 0x3bc28, 0x3bc28,
4126                 0x3bc3c, 0x3bc50,
4127                 0x3bcf0, 0x3bcfc,
4128                 0x3c000, 0x3c030,
4129                 0x3c100, 0x3c144,
4130                 0x3c190, 0x3c1d0,
4131                 0x3c200, 0x3c318,
4132                 0x3c400, 0x3c52c,
4133                 0x3c540, 0x3c61c,
4134                 0x3c800, 0x3c834,
4135                 0x3c8c0, 0x3c908,
4136                 0x3c910, 0x3c9ac,
4137                 0x3ca00, 0x3ca2c,
4138                 0x3ca44, 0x3ca50,
4139                 0x3ca74, 0x3cc24,
4140                 0x3cd00, 0x3cd00,
4141                 0x3cd08, 0x3cd14,
4142                 0x3cd1c, 0x3cd20,
4143                 0x3cd3c, 0x3cd50,
4144                 0x3d200, 0x3d20c,
4145                 0x3d220, 0x3d220,
4146                 0x3d240, 0x3d240,
4147                 0x3d600, 0x3d60c,
4148                 0x3da00, 0x3da1c,
4149                 0x3de00, 0x3de20,
4150                 0x3de38, 0x3de3c,
4151                 0x3de80, 0x3de80,
4152                 0x3de88, 0x3dea8,
4153                 0x3deb0, 0x3deb4,
4154                 0x3dec8, 0x3ded4,
4155                 0x3dfb8, 0x3e004,
4156                 0x3e200, 0x3e200,
4157                 0x3e208, 0x3e240,
4158                 0x3e248, 0x3e280,
4159                 0x3e288, 0x3e2c0,
4160                 0x3e2c8, 0x3e2fc,
4161                 0x3e600, 0x3e630,
4162                 0x3ea00, 0x3eabc,
4163                 0x3eb00, 0x3eb70,
4164                 0x3f000, 0x3f048,
4165                 0x3f060, 0x3f09c,
4166                 0x3f0f0, 0x3f148,
4167                 0x3f160, 0x3f19c,
4168                 0x3f1f0, 0x3f2e4,
4169                 0x3f2f8, 0x3f3e4,
4170                 0x3f3f8, 0x3f448,
4171                 0x3f460, 0x3f49c,
4172                 0x3f4f0, 0x3f548,
4173                 0x3f560, 0x3f59c,
4174                 0x3f5f0, 0x3f6e4,
4175                 0x3f6f8, 0x3f7e4,
4176                 0x3f7f8, 0x3f7fc,
4177                 0x3f814, 0x3f814,
4178                 0x3f82c, 0x3f82c,
4179                 0x3f880, 0x3f88c,
4180                 0x3f8e8, 0x3f8ec,
4181                 0x3f900, 0x3f948,
4182                 0x3f960, 0x3f99c,
4183                 0x3f9f0, 0x3fae4,
4184                 0x3faf8, 0x3fb10,
4185                 0x3fb28, 0x3fb28,
4186                 0x3fb3c, 0x3fb50,
4187                 0x3fbf0, 0x3fc10,
4188                 0x3fc28, 0x3fc28,
4189                 0x3fc3c, 0x3fc50,
4190                 0x3fcf0, 0x3fcfc,
4191                 0x40000, 0x4000c,
4192                 0x40040, 0x40068,
4193                 0x4007c, 0x40144,
4194                 0x40180, 0x4018c,
4195                 0x40200, 0x40298,
4196                 0x402ac, 0x4033c,
4197                 0x403f8, 0x403fc,
4198                 0x41304, 0x413c4,
4199                 0x41400, 0x4141c,
4200                 0x41480, 0x414d0,
4201                 0x44000, 0x44078,
4202                 0x440c0, 0x44278,
4203                 0x442c0, 0x44478,
4204                 0x444c0, 0x44678,
4205                 0x446c0, 0x44878,
4206                 0x448c0, 0x449fc,
4207                 0x45000, 0x45068,
4208                 0x45080, 0x45084,
4209                 0x450a0, 0x450b0,
4210                 0x45200, 0x45268,
4211                 0x45280, 0x45284,
4212                 0x452a0, 0x452b0,
4213                 0x460c0, 0x460e4,
4214                 0x47000, 0x4708c,
4215                 0x47200, 0x47250,
4216                 0x47400, 0x47420,
4217                 0x47600, 0x47618,
4218                 0x47800, 0x47814,
4219                 0x48000, 0x4800c,
4220                 0x48040, 0x48068,
4221                 0x4807c, 0x48144,
4222                 0x48180, 0x4818c,
4223                 0x48200, 0x48298,
4224                 0x482ac, 0x4833c,
4225                 0x483f8, 0x483fc,
4226                 0x49304, 0x493c4,
4227                 0x49400, 0x4941c,
4228                 0x49480, 0x494d0,
4229                 0x4c000, 0x4c078,
4230                 0x4c0c0, 0x4c278,
4231                 0x4c2c0, 0x4c478,
4232                 0x4c4c0, 0x4c678,
4233                 0x4c6c0, 0x4c878,
4234                 0x4c8c0, 0x4c9fc,
4235                 0x4d000, 0x4d068,
4236                 0x4d080, 0x4d084,
4237                 0x4d0a0, 0x4d0b0,
4238                 0x4d200, 0x4d268,
4239                 0x4d280, 0x4d284,
4240                 0x4d2a0, 0x4d2b0,
4241                 0x4e0c0, 0x4e0e4,
4242                 0x4f000, 0x4f08c,
4243                 0x4f200, 0x4f250,
4244                 0x4f400, 0x4f420,
4245                 0x4f600, 0x4f618,
4246                 0x4f800, 0x4f814,
4247                 0x50000, 0x500cc,
4248                 0x50400, 0x50400,
4249                 0x50800, 0x508cc,
4250                 0x50c00, 0x50c00,
4251                 0x51000, 0x5101c,
4252                 0x51300, 0x51308,
4253         };
4254
4255         if (is_t4(sc)) {
4256                 reg_ranges = &t4_reg_ranges[0];
4257                 n = nitems(t4_reg_ranges);
4258         } else {
4259                 reg_ranges = &t5_reg_ranges[0];
4260                 n = nitems(t5_reg_ranges);
4261         }
4262
4263         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4264         for (i = 0; i < n; i += 2)
4265                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4266 }
4267
4268 static void
4269 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4270 {
4271         struct ifnet *ifp = pi->ifp;
4272         struct sge_txq *txq;
4273         int i, drops;
4274         struct port_stats *s = &pi->stats;
4275         struct timeval tv;
4276         const struct timeval interval = {0, 250000};    /* 250ms */
4277
4278         getmicrotime(&tv);
4279         timevalsub(&tv, &interval);
4280         if (timevalcmp(&tv, &pi->last_refreshed, <))
4281                 return;
4282
4283         t4_get_port_stats(sc, pi->tx_chan, s);
4284
4285         ifp->if_opackets = s->tx_frames - s->tx_pause;
4286         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4287         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4288         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4289         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4290         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4291         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4292             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4293             s->rx_trunc3;
4294         for (i = 0; i < NCHAN; i++) {
4295                 if (pi->rx_chan_map & (1 << i)) {
4296                         uint32_t v;
4297
4298                         mtx_lock(&sc->regwin_lock);
4299                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4300                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4301                         mtx_unlock(&sc->regwin_lock);
4302                         ifp->if_iqdrops += v;
4303                 }
4304         }
4305
4306         drops = s->tx_drop;
4307         for_each_txq(pi, i, txq)
4308                 drops += counter_u64_fetch(txq->r->drops);
4309         ifp->if_snd.ifq_drops = drops;
4310
4311         ifp->if_oerrors = s->tx_error_frames;
4312         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4313             s->rx_fcs_err + s->rx_len_err;
4314
4315         getmicrotime(&pi->last_refreshed);
4316 }
4317
4318 static void
4319 cxgbe_tick(void *arg)
4320 {
4321         struct port_info *pi = arg;
4322         struct adapter *sc = pi->adapter;
4323         struct ifnet *ifp = pi->ifp;
4324
4325         PORT_LOCK(pi);
4326         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4327                 PORT_UNLOCK(pi);
4328                 return; /* without scheduling another callout */
4329         }
4330
4331         cxgbe_refresh_stats(sc, pi);
4332
4333         callout_schedule(&pi->tick, hz);
4334         PORT_UNLOCK(pi);
4335 }
4336
4337 static void
4338 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4339 {
4340         struct ifnet *vlan;
4341
4342         if (arg != ifp || ifp->if_type != IFT_ETHER)
4343                 return;
4344
4345         vlan = VLAN_DEVAT(ifp, vid);
4346         VLAN_SETCOOKIE(vlan, ifp);
4347 }
4348
4349 static int
4350 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4351 {
4352
4353 #ifdef INVARIANTS
4354         panic("%s: opcode 0x%02x on iq %p with payload %p",
4355             __func__, rss->opcode, iq, m);
4356 #else
4357         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4358             __func__, rss->opcode, iq, m);
4359         m_freem(m);
4360 #endif
4361         return (EDOOFUS);
4362 }
4363
4364 int
4365 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4366 {
4367         uintptr_t *loc, new;
4368
4369         if (opcode >= nitems(sc->cpl_handler))
4370                 return (EINVAL);
4371
4372         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4373         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4374         atomic_store_rel_ptr(loc, new);
4375
4376         return (0);
4377 }
4378
4379 static int
4380 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4381 {
4382
4383 #ifdef INVARIANTS
4384         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4385 #else
4386         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4387             __func__, iq, ctrl);
4388 #endif
4389         return (EDOOFUS);
4390 }
4391
4392 int
4393 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4394 {
4395         uintptr_t *loc, new;
4396
4397         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4398         loc = (uintptr_t *) &sc->an_handler;
4399         atomic_store_rel_ptr(loc, new);
4400
4401         return (0);
4402 }
4403
4404 static int
4405 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4406 {
4407         const struct cpl_fw6_msg *cpl =
4408             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4409
4410 #ifdef INVARIANTS
4411         panic("%s: fw_msg type %d", __func__, cpl->type);
4412 #else
4413         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4414 #endif
4415         return (EDOOFUS);
4416 }
4417
4418 int
4419 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4420 {
4421         uintptr_t *loc, new;
4422
4423         if (type >= nitems(sc->fw_msg_handler))
4424                 return (EINVAL);
4425
4426         /*
4427          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4428          * handler dispatch table.  Reject any attempt to install a handler for
4429          * this subtype.
4430          */
4431         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4432                 return (EINVAL);
4433
4434         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4435         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4436         atomic_store_rel_ptr(loc, new);
4437
4438         return (0);
4439 }
4440
4441 static int
4442 t4_sysctls(struct adapter *sc)
4443 {
4444         struct sysctl_ctx_list *ctx;
4445         struct sysctl_oid *oid;
4446         struct sysctl_oid_list *children, *c0;
4447         static char *caps[] = {
4448                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4449                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4450                     "\6HASHFILTER\7ETHOFLD",
4451                 "\20\1TOE",                             /* caps[2] toecaps */
4452                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4453                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4454                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4455                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4456                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4457                     "\4PO_INITIAOR\5PO_TARGET"
4458         };
4459         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4460
4461         ctx = device_get_sysctl_ctx(sc->dev);
4462
4463         /*
4464          * dev.t4nex.X.
4465          */
4466         oid = device_get_sysctl_tree(sc->dev);
4467         c0 = children = SYSCTL_CHILDREN(oid);
4468
4469         sc->sc_do_rxcopy = 1;
4470         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4471             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4472
4473         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4474             sc->params.nports, "# of ports");
4475
4476         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4477             NULL, chip_rev(sc), "chip hardware revision");
4478
4479         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4480             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4481
4482         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4483             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4484
4485         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4486             sc->cfcsum, "config file checksum");
4487
4488         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4489             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4490             sysctl_bitfield, "A", "available doorbells");
4491
4492         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4493             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4494             sysctl_bitfield, "A", "available link capabilities");
4495
4496         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4497             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4498             sysctl_bitfield, "A", "available NIC capabilities");
4499
4500         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4501             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4502             sysctl_bitfield, "A", "available TCP offload capabilities");
4503
4504         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4505             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4506             sysctl_bitfield, "A", "available RDMA capabilities");
4507
4508         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4509             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4510             sysctl_bitfield, "A", "available iSCSI capabilities");
4511
4512         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4513             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4514             sysctl_bitfield, "A", "available FCoE capabilities");
4515
4516         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4517             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4518
4519         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4520             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4521             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4522             "interrupt holdoff timer values (us)");
4523
4524         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4525             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4526             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4527             "interrupt holdoff packet counter values");
4528
4529         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4530             NULL, sc->tids.nftids, "number of filters");
4531
4532         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4533             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4534             "chip temperature (in Celsius)");
4535
4536         t4_sge_sysctls(sc, ctx, children);
4537
4538         sc->lro_timeout = 100;
4539         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4540             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4541
4542         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4543             &sc->debug_flags, 0, "flags to enable runtime debugging");
4544
4545 #ifdef SBUF_DRAIN
4546         /*
4547          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4548          */
4549         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4550             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4551             "logs and miscellaneous information");
4552         children = SYSCTL_CHILDREN(oid);
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4555             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4556             sysctl_cctrl, "A", "congestion control");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4559             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4560             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4563             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4564             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4565
4566         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4567             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4568             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4569
4570         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4571             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4572             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4575             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4576             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4580             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4581
4582         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4583             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584             sysctl_cim_la, "A", "CIM logic analyzer");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4592             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4596             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4600             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4604             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4608             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4612             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4613
4614         if (is_t5(sc)) {
4615                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4616                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4617                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4618
4619                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4620                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4621                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4622         }
4623
4624         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4625             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4626             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4627
4628         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4629             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4630             sysctl_cim_qcfg, "A", "CIM queue configuration");
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634             sysctl_cpl_stats, "A", "CPL statistics");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4642             sysctl_devlog, "A", "firmware's device log");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646             sysctl_fcoe_stats, "A", "FCoE statistics");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650             sysctl_hw_sched, "A", "hardware scheduler ");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4654             sysctl_l2t, "A", "hardware L2 table");
4655
4656         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4657             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4658             sysctl_lb_stats, "A", "loopback statistics");
4659
4660         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4661             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4662             sysctl_meminfo, "A", "memory regions");
4663
4664         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4665             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4666             sysctl_mps_tcam, "A", "MPS TCAM entries");
4667
4668         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4669             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4670             sysctl_path_mtus, "A", "path MTUs");
4671
4672         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4673             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4674             sysctl_pm_stats, "A", "PM statistics");
4675
4676         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4677             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4678             sysctl_rdma_stats, "A", "RDMA statistics");
4679
4680         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4681             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4682             sysctl_tcp_stats, "A", "TCP statistics");
4683
4684         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4685             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686             sysctl_tids, "A", "TID information");
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_tp_err_stats, "A", "TP error statistics");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_tp_la, "A", "TP logic analyzer");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_tx_rate, "A", "Tx rate");
4699
4700         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4701             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4702             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4703
4704         if (is_t5(sc)) {
4705                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4706                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4707                     sysctl_wcwr_stats, "A", "write combined work requests");
4708         }
4709 #endif
4710
4711 #ifdef TCP_OFFLOAD
4712         if (is_offload(sc)) {
4713                 /*
4714                  * dev.t4nex.X.toe.
4715                  */
4716                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4717                     NULL, "TOE parameters");
4718                 children = SYSCTL_CHILDREN(oid);
4719
4720                 sc->tt.sndbuf = 256 * 1024;
4721                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4722                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4723
4724                 sc->tt.ddp = 0;
4725                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4726                     &sc->tt.ddp, 0, "DDP allowed");
4727
4728                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4729                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4730                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4731
4732                 sc->tt.ddp_thres =
4733                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4734                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4735                     &sc->tt.ddp_thres, 0, "DDP threshold");
4736
4737                 sc->tt.rx_coalesce = 1;
4738                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4739                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4740
4741                 sc->tt.tx_align = 1;
4742                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4743                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4744         }
4745 #endif
4746
4747
4748         return (0);
4749 }
4750
4751 static int
4752 cxgbe_sysctls(struct port_info *pi)
4753 {
4754         struct sysctl_ctx_list *ctx;
4755         struct sysctl_oid *oid;
4756         struct sysctl_oid_list *children;
4757         struct adapter *sc = pi->adapter;
4758
4759         ctx = device_get_sysctl_ctx(pi->dev);
4760
4761         /*
4762          * dev.cxgbe.X.
4763          */
4764         oid = device_get_sysctl_tree(pi->dev);
4765         children = SYSCTL_CHILDREN(oid);
4766
4767         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4768            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4769         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4770                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4771                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4772                     "PHY temperature (in Celsius)");
4773                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4774                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4775                     "PHY firmware version");
4776         }
4777         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4778             &pi->nrxq, 0, "# of rx queues");
4779         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4780             &pi->ntxq, 0, "# of tx queues");
4781         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4782             &pi->first_rxq, 0, "index of first rx queue");
4783         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4784             &pi->first_txq, 0, "index of first tx queue");
4785         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4786             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4787             "Reserve queue 0 for non-flowid packets");
4788
4789 #ifdef TCP_OFFLOAD
4790         if (is_offload(sc)) {
4791                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4792                     &pi->nofldrxq, 0,
4793                     "# of rx queues for offloaded TCP connections");
4794                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4795                     &pi->nofldtxq, 0,
4796                     "# of tx queues for offloaded TCP connections");
4797                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4798                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4799                     "index of first TOE rx queue");
4800                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4801                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4802                     "index of first TOE tx queue");
4803         }
4804 #endif
4805 #ifdef DEV_NETMAP
4806         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4807             &pi->nnmrxq, 0, "# of rx queues for netmap");
4808         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4809             &pi->nnmtxq, 0, "# of tx queues for netmap");
4810         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4811             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4812             "index of first netmap rx queue");
4813         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4814             CTLFLAG_RD, &pi->first_nm_txq, 0,
4815             "index of first netmap tx queue");
4816 #endif
4817
4818         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4819             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4820             "holdoff timer index");
4821         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4822             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4823             "holdoff packet counter index");
4824
4825         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4826             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4827             "rx queue size");
4828         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4829             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4830             "tx queue size");
4831
4832         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4833             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4834             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4835
4836         /*
4837          * dev.cxgbe.X.stats.
4838          */
4839         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4840             NULL, "port statistics");
4841         children = SYSCTL_CHILDREN(oid);
4842         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4843             &pi->tx_parse_error, 0,
4844             "# of tx packets with invalid length or # of segments");
4845
4846 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4847         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4848             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4849             sysctl_handle_t4_reg64, "QU", desc)
4850
4851         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4853         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4855         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4857         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4858             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4859         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4860             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4861         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4862             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4863         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4864             "# of tx frames in this range",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4866         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4867             "# of tx frames in this range",
4868             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4869         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4870             "# of tx frames in this range",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4872         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4873             "# of tx frames in this range",
4874             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4875         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4876             "# of tx frames in this range",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4878         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4879             "# of tx frames in this range",
4880             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4881         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4882             "# of tx frames in this range",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4884         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4886         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4887             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4888         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4890         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4891             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4892         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4893             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4894         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4895             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4896         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4897             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4898         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4899             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4900         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4902         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4903             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4904
4905         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4913         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4915         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4917         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4919         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4920             "# of frames received with bad FCS",
4921             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4922         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4923             "# of frames received with length error",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4925         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4927         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4928             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4929         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4930             "# of rx frames in this range",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4932         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4933             "# of rx frames in this range",
4934             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4935         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4936             "# of rx frames in this range",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4938         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4939             "# of rx frames in this range",
4940             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4941         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4942             "# of rx frames in this range",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4944         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4945             "# of rx frames in this range",
4946             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4947         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4948             "# of rx frames in this range",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4950         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4952         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4954         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4956         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4958         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4960         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4962         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4964         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4966         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4968
4969 #undef SYSCTL_ADD_T4_REG64
4970
4971 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4972         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4973             &pi->stats.name, desc)
4974
4975         /* We get these from port_stats and they may be stale by upto 1s */
4976         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4977             "# drops due to buffer-group 0 overflows");
4978         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4979             "# drops due to buffer-group 1 overflows");
4980         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4981             "# drops due to buffer-group 2 overflows");
4982         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4983             "# drops due to buffer-group 3 overflows");
4984         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4985             "# of buffer-group 0 truncated packets");
4986         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4987             "# of buffer-group 1 truncated packets");
4988         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4989             "# of buffer-group 2 truncated packets");
4990         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4991             "# of buffer-group 3 truncated packets");
4992
4993 #undef SYSCTL_ADD_T4_PORTSTAT
4994
4995         return (0);
4996 }
4997
4998 static int
4999 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5000 {
5001         int rc, *i, space = 0;
5002         struct sbuf sb;
5003
5004         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5005         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5006                 if (space)
5007                         sbuf_printf(&sb, " ");
5008                 sbuf_printf(&sb, "%d", *i);
5009                 space = 1;
5010         }
5011         sbuf_finish(&sb);
5012         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5013         sbuf_delete(&sb);
5014         return (rc);
5015 }
5016
5017 static int
5018 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5019 {
5020         int rc;
5021         struct sbuf *sb;
5022
5023         rc = sysctl_wire_old_buffer(req, 0);
5024         if (rc != 0)
5025                 return(rc);
5026
5027         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5028         if (sb == NULL)
5029                 return (ENOMEM);
5030
5031         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5032         rc = sbuf_finish(sb);
5033         sbuf_delete(sb);
5034
5035         return (rc);
5036 }
5037
5038 static int
5039 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5040 {
5041         struct port_info *pi = arg1;
5042         int op = arg2;
5043         struct adapter *sc = pi->adapter;
5044         u_int v;
5045         int rc;
5046
5047         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5048         if (rc)
5049                 return (rc);
5050         /* XXX: magic numbers */
5051         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5052             &v);
5053         end_synchronized_op(sc, 0);
5054         if (rc)
5055                 return (rc);
5056         if (op == 0)
5057                 v /= 256;
5058
5059         rc = sysctl_handle_int(oidp, &v, 0, req);
5060         return (rc);
5061 }
5062
5063 static int
5064 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5065 {
5066         struct port_info *pi = arg1;
5067         int rc, val;
5068
5069         val = pi->rsrv_noflowq;
5070         rc = sysctl_handle_int(oidp, &val, 0, req);
5071         if (rc != 0 || req->newptr == NULL)
5072                 return (rc);
5073
5074         if ((val >= 1) && (pi->ntxq > 1))
5075                 pi->rsrv_noflowq = 1;
5076         else
5077                 pi->rsrv_noflowq = 0;
5078
5079         return (rc);
5080 }
5081
5082 static int
5083 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5084 {
5085         struct port_info *pi = arg1;
5086         struct adapter *sc = pi->adapter;
5087         int idx, rc, i;
5088         struct sge_rxq *rxq;
5089 #ifdef TCP_OFFLOAD
5090         struct sge_ofld_rxq *ofld_rxq;
5091 #endif
5092         uint8_t v;
5093
5094         idx = pi->tmr_idx;
5095
5096         rc = sysctl_handle_int(oidp, &idx, 0, req);
5097         if (rc != 0 || req->newptr == NULL)
5098                 return (rc);
5099
5100         if (idx < 0 || idx >= SGE_NTIMERS)
5101                 return (EINVAL);
5102
5103         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5104             "t4tmr");
5105         if (rc)
5106                 return (rc);
5107
5108         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5109         for_each_rxq(pi, i, rxq) {
5110 #ifdef atomic_store_rel_8
5111                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5112 #else
5113                 rxq->iq.intr_params = v;
5114 #endif
5115         }
5116 #ifdef TCP_OFFLOAD
5117         for_each_ofld_rxq(pi, i, ofld_rxq) {
5118 #ifdef atomic_store_rel_8
5119                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5120 #else
5121                 ofld_rxq->iq.intr_params = v;
5122 #endif
5123         }
5124 #endif
5125         pi->tmr_idx = idx;
5126
5127         end_synchronized_op(sc, LOCK_HELD);
5128         return (0);
5129 }
5130
5131 static int
5132 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5133 {
5134         struct port_info *pi = arg1;
5135         struct adapter *sc = pi->adapter;
5136         int idx, rc;
5137
5138         idx = pi->pktc_idx;
5139
5140         rc = sysctl_handle_int(oidp, &idx, 0, req);
5141         if (rc != 0 || req->newptr == NULL)
5142                 return (rc);
5143
5144         if (idx < -1 || idx >= SGE_NCOUNTERS)
5145                 return (EINVAL);
5146
5147         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5148             "t4pktc");
5149         if (rc)
5150                 return (rc);
5151
5152         if (pi->flags & PORT_INIT_DONE)
5153                 rc = EBUSY; /* cannot be changed once the queues are created */
5154         else
5155                 pi->pktc_idx = idx;
5156
5157         end_synchronized_op(sc, LOCK_HELD);
5158         return (rc);
5159 }
5160
5161 static int
5162 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5163 {
5164         struct port_info *pi = arg1;
5165         struct adapter *sc = pi->adapter;
5166         int qsize, rc;
5167
5168         qsize = pi->qsize_rxq;
5169
5170         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5171         if (rc != 0 || req->newptr == NULL)
5172                 return (rc);
5173
5174         if (qsize < 128 || (qsize & 7))
5175                 return (EINVAL);
5176
5177         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5178             "t4rxqs");
5179         if (rc)
5180                 return (rc);
5181
5182         if (pi->flags & PORT_INIT_DONE)
5183                 rc = EBUSY; /* cannot be changed once the queues are created */
5184         else
5185                 pi->qsize_rxq = qsize;
5186
5187         end_synchronized_op(sc, LOCK_HELD);
5188         return (rc);
5189 }
5190
5191 static int
5192 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5193 {
5194         struct port_info *pi = arg1;
5195         struct adapter *sc = pi->adapter;
5196         int qsize, rc;
5197
5198         qsize = pi->qsize_txq;
5199
5200         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5201         if (rc != 0 || req->newptr == NULL)
5202                 return (rc);
5203
5204         if (qsize < 128 || qsize > 65536)
5205                 return (EINVAL);
5206
5207         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5208             "t4txqs");
5209         if (rc)
5210                 return (rc);
5211
5212         if (pi->flags & PORT_INIT_DONE)
5213                 rc = EBUSY; /* cannot be changed once the queues are created */
5214         else
5215                 pi->qsize_txq = qsize;
5216
5217         end_synchronized_op(sc, LOCK_HELD);
5218         return (rc);
5219 }
5220
5221 static int
5222 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5223 {
5224         struct port_info *pi = arg1;
5225         struct adapter *sc = pi->adapter;
5226         struct link_config *lc = &pi->link_cfg;
5227         int rc;
5228
5229         if (req->newptr == NULL) {
5230                 struct sbuf *sb;
5231                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5232
5233                 rc = sysctl_wire_old_buffer(req, 0);
5234                 if (rc != 0)
5235                         return(rc);
5236
5237                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5238                 if (sb == NULL)
5239                         return (ENOMEM);
5240
5241                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5242                 rc = sbuf_finish(sb);
5243                 sbuf_delete(sb);
5244         } else {
5245                 char s[2];
5246                 int n;
5247
5248                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5249                 s[1] = 0;
5250
5251                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5252                 if (rc != 0)
5253                         return(rc);
5254
5255                 if (s[1] != 0)
5256                         return (EINVAL);
5257                 if (s[0] < '0' || s[0] > '9')
5258                         return (EINVAL);        /* not a number */
5259                 n = s[0] - '0';
5260                 if (n & ~(PAUSE_TX | PAUSE_RX))
5261                         return (EINVAL);        /* some other bit is set too */
5262
5263                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5264                 if (rc)
5265                         return (rc);
5266                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5267                         int link_ok = lc->link_ok;
5268
5269                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5270                         lc->requested_fc |= n;
5271                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5272                         lc->link_ok = link_ok;  /* restore */
5273                 }
5274                 end_synchronized_op(sc, 0);
5275         }
5276
5277         return (rc);
5278 }
5279
5280 static int
5281 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5282 {
5283         struct adapter *sc = arg1;
5284         int reg = arg2;
5285         uint64_t val;
5286
5287         val = t4_read_reg64(sc, reg);
5288
5289         return (sysctl_handle_64(oidp, &val, 0, req));
5290 }
5291
5292 static int
5293 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5294 {
5295         struct adapter *sc = arg1;
5296         int rc, t;
5297         uint32_t param, val;
5298
5299         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5300         if (rc)
5301                 return (rc);
5302         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5303             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5304             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5305         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5306         end_synchronized_op(sc, 0);
5307         if (rc)
5308                 return (rc);
5309
5310         /* unknown is returned as 0 but we display -1 in that case */
5311         t = val == 0 ? -1 : val;
5312
5313         rc = sysctl_handle_int(oidp, &t, 0, req);
5314         return (rc);
5315 }
5316
5317 #ifdef SBUF_DRAIN
5318 static int
5319 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5320 {
5321         struct adapter *sc = arg1;
5322         struct sbuf *sb;
5323         int rc, i;
5324         uint16_t incr[NMTUS][NCCTRL_WIN];
5325         static const char *dec_fac[] = {
5326                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5327                 "0.9375"
5328         };
5329
5330         rc = sysctl_wire_old_buffer(req, 0);
5331         if (rc != 0)
5332                 return (rc);
5333
5334         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5335         if (sb == NULL)
5336                 return (ENOMEM);
5337
5338         t4_read_cong_tbl(sc, incr);
5339
5340         for (i = 0; i < NCCTRL_WIN; ++i) {
5341                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5342                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5343                     incr[5][i], incr[6][i], incr[7][i]);
5344                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5345                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5346                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5347                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5348         }
5349
5350         rc = sbuf_finish(sb);
5351         sbuf_delete(sb);
5352
5353         return (rc);
5354 }
5355
5356 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5357         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5358         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5359         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5360 };
5361
5362 static int
5363 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5364 {
5365         struct adapter *sc = arg1;
5366         struct sbuf *sb;
5367         int rc, i, n, qid = arg2;
5368         uint32_t *buf, *p;
5369         char *qtype;
5370         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5371
5372         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5373             ("%s: bad qid %d\n", __func__, qid));
5374
5375         if (qid < CIM_NUM_IBQ) {
5376                 /* inbound queue */
5377                 qtype = "IBQ";
5378                 n = 4 * CIM_IBQ_SIZE;
5379                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5380                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5381         } else {
5382                 /* outbound queue */
5383                 qtype = "OBQ";
5384                 qid -= CIM_NUM_IBQ;
5385                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5386                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5387                 rc = t4_read_cim_obq(sc, qid, buf, n);
5388         }
5389
5390         if (rc < 0) {
5391                 rc = -rc;
5392                 goto done;
5393         }
5394         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5395
5396         rc = sysctl_wire_old_buffer(req, 0);
5397         if (rc != 0)
5398                 goto done;
5399
5400         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5401         if (sb == NULL) {
5402                 rc = ENOMEM;
5403                 goto done;
5404         }
5405
5406         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5407         for (i = 0, p = buf; i < n; i += 16, p += 4)
5408                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5409                     p[2], p[3]);
5410
5411         rc = sbuf_finish(sb);
5412         sbuf_delete(sb);
5413 done:
5414         free(buf, M_CXGBE);
5415         return (rc);
5416 }
5417
5418 static int
5419 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5420 {
5421         struct adapter *sc = arg1;
5422         u_int cfg;
5423         struct sbuf *sb;
5424         uint32_t *buf, *p;
5425         int rc;
5426
5427         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5428         if (rc != 0)
5429                 return (rc);
5430
5431         rc = sysctl_wire_old_buffer(req, 0);
5432         if (rc != 0)
5433                 return (rc);
5434
5435         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5436         if (sb == NULL)
5437                 return (ENOMEM);
5438
5439         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5440             M_ZERO | M_WAITOK);
5441
5442         rc = -t4_cim_read_la(sc, buf, NULL);
5443         if (rc != 0)
5444                 goto done;
5445
5446         sbuf_printf(sb, "Status   Data      PC%s",
5447             cfg & F_UPDBGLACAPTPCONLY ? "" :
5448             "     LS0Stat  LS0Addr             LS0Data");
5449
5450         KASSERT((sc->params.cim_la_size & 7) == 0,
5451             ("%s: p will walk off the end of buf", __func__));
5452
5453         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5454                 if (cfg & F_UPDBGLACAPTPCONLY) {
5455                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5456                             p[6], p[7]);
5457                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5458                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5459                             p[4] & 0xff, p[5] >> 8);
5460                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5461                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5462                             p[1] & 0xf, p[2] >> 4);
5463                 } else {
5464                         sbuf_printf(sb,
5465                             "\n  %02x   %x%07x %x%07x %08x %08x "
5466                             "%08x%08x%08x%08x",
5467                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5468                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5469                             p[6], p[7]);
5470                 }
5471         }
5472
5473         rc = sbuf_finish(sb);
5474         sbuf_delete(sb);
5475 done:
5476         free(buf, M_CXGBE);
5477         return (rc);
5478 }
5479
5480 static int
5481 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5482 {
5483         struct adapter *sc = arg1;
5484         u_int i;
5485         struct sbuf *sb;
5486         uint32_t *buf, *p;
5487         int rc;
5488
5489         rc = sysctl_wire_old_buffer(req, 0);
5490         if (rc != 0)
5491                 return (rc);
5492
5493         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5494         if (sb == NULL)
5495                 return (ENOMEM);
5496
5497         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5498             M_ZERO | M_WAITOK);
5499
5500         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5501         p = buf;
5502
5503         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5504                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5505                     p[1], p[0]);
5506         }
5507
5508         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5509         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5510                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5511                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5512                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5513                     (p[1] >> 2) | ((p[2] & 3) << 30),
5514                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5515                     p[0] & 1);
5516         }
5517
5518         rc = sbuf_finish(sb);
5519         sbuf_delete(sb);
5520         free(buf, M_CXGBE);
5521         return (rc);
5522 }
5523
5524 static int
5525 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5526 {
5527         struct adapter *sc = arg1;
5528         u_int i;
5529         struct sbuf *sb;
5530         uint32_t *buf, *p;
5531         int rc;
5532
5533         rc = sysctl_wire_old_buffer(req, 0);
5534         if (rc != 0)
5535                 return (rc);
5536
5537         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5538         if (sb == NULL)
5539                 return (ENOMEM);
5540
5541         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5542             M_ZERO | M_WAITOK);
5543
5544         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5545         p = buf;
5546
5547         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5548         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5549                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5550                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5551                     p[4], p[3], p[2], p[1], p[0]);
5552         }
5553
5554         sbuf_printf(sb, "\n\nCntl ID               Data");
5555         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5556                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5557                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5558         }
5559
5560         rc = sbuf_finish(sb);
5561         sbuf_delete(sb);
5562         free(buf, M_CXGBE);
5563         return (rc);
5564 }
5565
5566 static int
5567 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5568 {
5569         struct adapter *sc = arg1;
5570         struct sbuf *sb;
5571         int rc, i;
5572         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5573         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5574         uint16_t thres[CIM_NUM_IBQ];
5575         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5576         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5577         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5578
5579         if (is_t4(sc)) {
5580                 cim_num_obq = CIM_NUM_OBQ;
5581                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5582                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5583         } else {
5584                 cim_num_obq = CIM_NUM_OBQ_T5;
5585                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5586                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5587         }
5588         nq = CIM_NUM_IBQ + cim_num_obq;
5589
5590         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5591         if (rc == 0)
5592                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5593         if (rc != 0)
5594                 return (rc);
5595
5596         t4_read_cimq_cfg(sc, base, size, thres);
5597
5598         rc = sysctl_wire_old_buffer(req, 0);
5599         if (rc != 0)
5600                 return (rc);
5601
5602         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5603         if (sb == NULL)
5604                 return (ENOMEM);
5605
5606         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5607
5608         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5609                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5610                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5611                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5612                     G_QUEREMFLITS(p[2]) * 16);
5613         for ( ; i < nq; i++, p += 4, wr += 2)
5614                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5615                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5616                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5617                     G_QUEREMFLITS(p[2]) * 16);
5618
5619         rc = sbuf_finish(sb);
5620         sbuf_delete(sb);
5621
5622         return (rc);
5623 }
5624
5625 static int
5626 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5627 {
5628         struct adapter *sc = arg1;
5629         struct sbuf *sb;
5630         int rc;
5631         struct tp_cpl_stats stats;
5632
5633         rc = sysctl_wire_old_buffer(req, 0);
5634         if (rc != 0)
5635                 return (rc);
5636
5637         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5638         if (sb == NULL)
5639                 return (ENOMEM);
5640
5641         t4_tp_get_cpl_stats(sc, &stats);
5642
5643         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5644             "channel 3\n");
5645         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5646                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5647         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5648                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5649
5650         rc = sbuf_finish(sb);
5651         sbuf_delete(sb);
5652
5653         return (rc);
5654 }
5655
5656 static int
5657 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5658 {
5659         struct adapter *sc = arg1;
5660         struct sbuf *sb;
5661         int rc;
5662         struct tp_usm_stats stats;
5663
5664         rc = sysctl_wire_old_buffer(req, 0);
5665         if (rc != 0)
5666                 return(rc);
5667
5668         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5669         if (sb == NULL)
5670                 return (ENOMEM);
5671
5672         t4_get_usm_stats(sc, &stats);
5673
5674         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5675         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5676         sbuf_printf(sb, "Drops:  %u", stats.drops);
5677
5678         rc = sbuf_finish(sb);
5679         sbuf_delete(sb);
5680
5681         return (rc);
5682 }
5683
5684 const char *devlog_level_strings[] = {
5685         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5686         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5687         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5688         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5689         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5690         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5691 };
5692
5693 const char *devlog_facility_strings[] = {
5694         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5695         [FW_DEVLOG_FACILITY_CF]         = "CF",
5696         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5697         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5698         [FW_DEVLOG_FACILITY_RES]        = "RES",
5699         [FW_DEVLOG_FACILITY_HW]         = "HW",
5700         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5701         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5702         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5703         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5704         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5705         [FW_DEVLOG_FACILITY_VI]         = "VI",
5706         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5707         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5708         [FW_DEVLOG_FACILITY_TM]         = "TM",
5709         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5710         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5711         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5712         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5713         [FW_DEVLOG_FACILITY_RI]         = "RI",
5714         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5715         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5716         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5717         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5718 };
5719
5720 static int
5721 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5722 {
5723         struct adapter *sc = arg1;
5724         struct devlog_params *dparams = &sc->params.devlog;
5725         struct fw_devlog_e *buf, *e;
5726         int i, j, rc, nentries, first = 0, m;
5727         struct sbuf *sb;
5728         uint64_t ftstamp = UINT64_MAX;
5729
5730         if (dparams->start == 0) {
5731                 dparams->memtype = FW_MEMTYPE_EDC0;
5732                 dparams->start = 0x84000;
5733                 dparams->size = 32768;
5734         }
5735
5736         nentries = dparams->size / sizeof(struct fw_devlog_e);
5737
5738         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5739         if (buf == NULL)
5740                 return (ENOMEM);
5741
5742         m = fwmtype_to_hwmtype(dparams->memtype);
5743         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5744         if (rc != 0)
5745                 goto done;
5746
5747         for (i = 0; i < nentries; i++) {
5748                 e = &buf[i];
5749
5750                 if (e->timestamp == 0)
5751                         break;  /* end */
5752
5753                 e->timestamp = be64toh(e->timestamp);
5754                 e->seqno = be32toh(e->seqno);
5755                 for (j = 0; j < 8; j++)
5756                         e->params[j] = be32toh(e->params[j]);
5757
5758                 if (e->timestamp < ftstamp) {
5759                         ftstamp = e->timestamp;
5760                         first = i;
5761                 }
5762         }
5763
5764         if (buf[first].timestamp == 0)
5765                 goto done;      /* nothing in the log */
5766
5767         rc = sysctl_wire_old_buffer(req, 0);
5768         if (rc != 0)
5769                 goto done;
5770
5771         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5772         if (sb == NULL) {
5773                 rc = ENOMEM;
5774                 goto done;
5775         }
5776         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5777             "Seq#", "Tstamp", "Level", "Facility", "Message");
5778
5779         i = first;
5780         do {
5781                 e = &buf[i];
5782                 if (e->timestamp == 0)
5783                         break;  /* end */
5784
5785                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5786                     e->seqno, e->timestamp,
5787                     (e->level < nitems(devlog_level_strings) ?
5788                         devlog_level_strings[e->level] : "UNKNOWN"),
5789                     (e->facility < nitems(devlog_facility_strings) ?
5790                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5791                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5792                     e->params[2], e->params[3], e->params[4],
5793                     e->params[5], e->params[6], e->params[7]);
5794
5795                 if (++i == nentries)
5796                         i = 0;
5797         } while (i != first);
5798
5799         rc = sbuf_finish(sb);
5800         sbuf_delete(sb);
5801 done:
5802         free(buf, M_CXGBE);
5803         return (rc);
5804 }
5805
5806 static int
5807 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5808 {
5809         struct adapter *sc = arg1;
5810         struct sbuf *sb;
5811         int rc;
5812         struct tp_fcoe_stats stats[4];
5813
5814         rc = sysctl_wire_old_buffer(req, 0);
5815         if (rc != 0)
5816                 return (rc);
5817
5818         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5819         if (sb == NULL)
5820                 return (ENOMEM);
5821
5822         t4_get_fcoe_stats(sc, 0, &stats[0]);
5823         t4_get_fcoe_stats(sc, 1, &stats[1]);
5824         t4_get_fcoe_stats(sc, 2, &stats[2]);
5825         t4_get_fcoe_stats(sc, 3, &stats[3]);
5826
5827         sbuf_printf(sb, "                   channel 0        channel 1        "
5828             "channel 2        channel 3\n");
5829         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5830             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5831             stats[3].octetsDDP);
5832         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5833             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5834         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5835             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5836             stats[3].framesDrop);
5837
5838         rc = sbuf_finish(sb);
5839         sbuf_delete(sb);
5840
5841         return (rc);
5842 }
5843
5844 static int
5845 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5846 {
5847         struct adapter *sc = arg1;
5848         struct sbuf *sb;
5849         int rc, i;
5850         unsigned int map, kbps, ipg, mode;
5851         unsigned int pace_tab[NTX_SCHED];
5852
5853         rc = sysctl_wire_old_buffer(req, 0);
5854         if (rc != 0)
5855                 return (rc);
5856
5857         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5858         if (sb == NULL)
5859                 return (ENOMEM);
5860
5861         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5862         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5863         t4_read_pace_tbl(sc, pace_tab);
5864
5865         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5866             "Class IPG (0.1 ns)   Flow IPG (us)");
5867
5868         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5869                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5870                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5871                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5872                 if (kbps)
5873                         sbuf_printf(sb, "%9u     ", kbps);
5874                 else
5875                         sbuf_printf(sb, " disabled     ");
5876
5877                 if (ipg)
5878                         sbuf_printf(sb, "%13u        ", ipg);
5879                 else
5880                         sbuf_printf(sb, "     disabled        ");
5881
5882                 if (pace_tab[i])
5883                         sbuf_printf(sb, "%10u", pace_tab[i]);
5884                 else
5885                         sbuf_printf(sb, "  disabled");
5886         }
5887
5888         rc = sbuf_finish(sb);
5889         sbuf_delete(sb);
5890
5891         return (rc);
5892 }
5893
5894 static int
5895 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5896 {
5897         struct adapter *sc = arg1;
5898         struct sbuf *sb;
5899         int rc, i, j;
5900         uint64_t *p0, *p1;
5901         struct lb_port_stats s[2];
5902         static const char *stat_name[] = {
5903                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5904                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5905                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5906                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5907                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5908                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5909                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5910         };
5911
5912         rc = sysctl_wire_old_buffer(req, 0);
5913         if (rc != 0)
5914                 return (rc);
5915
5916         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5917         if (sb == NULL)
5918                 return (ENOMEM);
5919
5920         memset(s, 0, sizeof(s));
5921
5922         for (i = 0; i < 4; i += 2) {
5923                 t4_get_lb_stats(sc, i, &s[0]);
5924                 t4_get_lb_stats(sc, i + 1, &s[1]);
5925
5926                 p0 = &s[0].octets;
5927                 p1 = &s[1].octets;
5928                 sbuf_printf(sb, "%s                       Loopback %u"
5929                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5930
5931                 for (j = 0; j < nitems(stat_name); j++)
5932                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5933                                    *p0++, *p1++);
5934         }
5935
5936         rc = sbuf_finish(sb);
5937         sbuf_delete(sb);
5938
5939         return (rc);
5940 }
5941
5942 static int
5943 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5944 {
5945         int rc = 0;
5946         struct port_info *pi = arg1;
5947         struct sbuf *sb;
5948         static const char *linkdnreasons[] = {
5949                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5950                 "PHY overheated", "unknown", "rx los", "reserved7"
5951         };
5952
5953         rc = sysctl_wire_old_buffer(req, 0);
5954         if (rc != 0)
5955                 return(rc);
5956         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5957         if (sb == NULL)
5958                 return (ENOMEM);
5959
5960         if (pi->linkdnrc < 0)
5961                 sbuf_printf(sb, "n/a");
5962         else if (pi->linkdnrc < nitems(linkdnreasons))
5963                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5964         else
5965                 sbuf_printf(sb, "%d", pi->linkdnrc);
5966
5967         rc = sbuf_finish(sb);
5968         sbuf_delete(sb);
5969
5970         return (rc);
5971 }
5972
5973 struct mem_desc {
5974         unsigned int base;
5975         unsigned int limit;
5976         unsigned int idx;
5977 };
5978
5979 static int
5980 mem_desc_cmp(const void *a, const void *b)
5981 {
5982         return ((const struct mem_desc *)a)->base -
5983                ((const struct mem_desc *)b)->base;
5984 }
5985
5986 static void
5987 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5988     unsigned int to)
5989 {
5990         unsigned int size;
5991
5992         size = to - from + 1;
5993         if (size == 0)
5994                 return;
5995
5996         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5997         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5998 }
5999
6000 static int
6001 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6002 {
6003         struct adapter *sc = arg1;
6004         struct sbuf *sb;
6005         int rc, i, n;
6006         uint32_t lo, hi, used, alloc;
6007         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6008         static const char *region[] = {
6009                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6010                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6011                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6012                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6013                 "RQUDP region:", "PBL region:", "TXPBL region:",
6014                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6015                 "On-chip queues:"
6016         };
6017         struct mem_desc avail[4];
6018         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6019         struct mem_desc *md = mem;
6020
6021         rc = sysctl_wire_old_buffer(req, 0);
6022         if (rc != 0)
6023                 return (rc);
6024
6025         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6026         if (sb == NULL)
6027                 return (ENOMEM);
6028
6029         for (i = 0; i < nitems(mem); i++) {
6030                 mem[i].limit = 0;
6031                 mem[i].idx = i;
6032         }
6033
6034         /* Find and sort the populated memory ranges */
6035         i = 0;
6036         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6037         if (lo & F_EDRAM0_ENABLE) {
6038                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6039                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6040                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6041                 avail[i].idx = 0;
6042                 i++;
6043         }
6044         if (lo & F_EDRAM1_ENABLE) {
6045                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6046                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6047                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6048                 avail[i].idx = 1;
6049                 i++;
6050         }
6051         if (lo & F_EXT_MEM_ENABLE) {
6052                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6053                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6054                 avail[i].limit = avail[i].base +
6055                     (G_EXT_MEM_SIZE(hi) << 20);
6056                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6057                 i++;
6058         }
6059         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6060                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6061                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6062                 avail[i].limit = avail[i].base +
6063                     (G_EXT_MEM1_SIZE(hi) << 20);
6064                 avail[i].idx = 4;
6065                 i++;
6066         }
6067         if (!i)                                    /* no memory available */
6068                 return 0;
6069         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6070
6071         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6072         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6073         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6074         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6075         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6076         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6077         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6078         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6079         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6080
6081         /* the next few have explicit upper bounds */
6082         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6083         md->limit = md->base - 1 +
6084                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6085                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6086         md++;
6087
6088         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6089         md->limit = md->base - 1 +
6090                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6091                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6092         md++;
6093
6094         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6095                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6096                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6097                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6098         } else {
6099                 md->base = 0;
6100                 md->idx = nitems(region);  /* hide it */
6101         }
6102         md++;
6103
6104 #define ulp_region(reg) \
6105         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6106         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6107
6108         ulp_region(RX_ISCSI);
6109         ulp_region(RX_TDDP);
6110         ulp_region(TX_TPT);
6111         ulp_region(RX_STAG);
6112         ulp_region(RX_RQ);
6113         ulp_region(RX_RQUDP);
6114         ulp_region(RX_PBL);
6115         ulp_region(TX_PBL);
6116 #undef ulp_region
6117
6118         md->base = 0;
6119         md->idx = nitems(region);
6120         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6121                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6122                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6123                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6124         }
6125         md++;
6126
6127         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6128         md->limit = md->base + sc->tids.ntids - 1;
6129         md++;
6130         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6131         md->limit = md->base + sc->tids.ntids - 1;
6132         md++;
6133
6134         md->base = sc->vres.ocq.start;
6135         if (sc->vres.ocq.size)
6136                 md->limit = md->base + sc->vres.ocq.size - 1;
6137         else
6138                 md->idx = nitems(region);  /* hide it */
6139         md++;
6140
6141         /* add any address-space holes, there can be up to 3 */
6142         for (n = 0; n < i - 1; n++)
6143                 if (avail[n].limit < avail[n + 1].base)
6144                         (md++)->base = avail[n].limit;
6145         if (avail[n].limit)
6146                 (md++)->base = avail[n].limit;
6147
6148         n = md - mem;
6149         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6150
6151         for (lo = 0; lo < i; lo++)
6152                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6153                                 avail[lo].limit - 1);
6154
6155         sbuf_printf(sb, "\n");
6156         for (i = 0; i < n; i++) {
6157                 if (mem[i].idx >= nitems(region))
6158                         continue;                        /* skip holes */
6159                 if (!mem[i].limit)
6160                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6161                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6162                                 mem[i].limit);
6163         }
6164
6165         sbuf_printf(sb, "\n");
6166         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6167         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6168         mem_region_show(sb, "uP RAM:", lo, hi);
6169
6170         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6171         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6172         mem_region_show(sb, "uP Extmem2:", lo, hi);
6173
6174         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6175         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6176                    G_PMRXMAXPAGE(lo),
6177                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6178                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6179
6180         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6181         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6182         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6183                    G_PMTXMAXPAGE(lo),
6184                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6185                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6186         sbuf_printf(sb, "%u p-structs\n",
6187                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6188
6189         for (i = 0; i < 4; i++) {
6190                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6191                 if (is_t4(sc)) {
6192                         used = G_USED(lo);
6193                         alloc = G_ALLOC(lo);
6194                 } else {
6195                         used = G_T5_USED(lo);
6196                         alloc = G_T5_ALLOC(lo);
6197                 }
6198                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6199                            i, used, alloc);
6200         }
6201         for (i = 0; i < 4; i++) {
6202                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6203                 if (is_t4(sc)) {
6204                         used = G_USED(lo);
6205                         alloc = G_ALLOC(lo);
6206                 } else {
6207                         used = G_T5_USED(lo);
6208                         alloc = G_T5_ALLOC(lo);
6209                 }
6210                 sbuf_printf(sb,
6211                            "\nLoopback %d using %u pages out of %u allocated",
6212                            i, used, alloc);
6213         }
6214
6215         rc = sbuf_finish(sb);
6216         sbuf_delete(sb);
6217
6218         return (rc);
6219 }
6220
6221 static inline void
6222 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6223 {
6224         *mask = x | y;
6225         y = htobe64(y);
6226         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6227 }
6228
6229 static int
6230 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6231 {
6232         struct adapter *sc = arg1;
6233         struct sbuf *sb;
6234         int rc, i, n;
6235
6236         rc = sysctl_wire_old_buffer(req, 0);
6237         if (rc != 0)
6238                 return (rc);
6239
6240         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6241         if (sb == NULL)
6242                 return (ENOMEM);
6243
6244         sbuf_printf(sb,
6245             "Idx  Ethernet address     Mask     Vld Ports PF"
6246             "  VF              Replication             P0 P1 P2 P3  ML");
6247         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6248             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6249         for (i = 0; i < n; i++) {
6250                 uint64_t tcamx, tcamy, mask;
6251                 uint32_t cls_lo, cls_hi;
6252                 uint8_t addr[ETHER_ADDR_LEN];
6253
6254                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6255                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6256                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6257                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6258
6259                 if (tcamx & tcamy)
6260                         continue;
6261
6262                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6263                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6264                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6265                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6266                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6267                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6268                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6269
6270                 if (cls_lo & F_REPLICATE) {
6271                         struct fw_ldst_cmd ldst_cmd;
6272
6273                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6274                         ldst_cmd.op_to_addrspace =
6275                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6276                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6277                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6278                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6279                         ldst_cmd.u.mps.rplc.fid_idx =
6280                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6281                                 V_FW_LDST_CMD_IDX(i));
6282
6283                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6284                             "t4mps");
6285                         if (rc)
6286                                 break;
6287                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6288                             sizeof(ldst_cmd), &ldst_cmd);
6289                         end_synchronized_op(sc, 0);
6290
6291                         if (rc != 0) {
6292                                 sbuf_printf(sb,
6293                                     " ------------ error %3u ------------", rc);
6294                                 rc = 0;
6295                         } else {
6296                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6297                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6298                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6299                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6300                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6301                         }
6302                 } else
6303                         sbuf_printf(sb, "%36s", "");
6304
6305                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6306                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6307                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6308         }
6309
6310         if (rc)
6311                 (void) sbuf_finish(sb);
6312         else
6313                 rc = sbuf_finish(sb);
6314         sbuf_delete(sb);
6315
6316         return (rc);
6317 }
6318
6319 static int
6320 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6321 {
6322         struct adapter *sc = arg1;
6323         struct sbuf *sb;
6324         int rc;
6325         uint16_t mtus[NMTUS];
6326
6327         rc = sysctl_wire_old_buffer(req, 0);
6328         if (rc != 0)
6329                 return (rc);
6330
6331         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6332         if (sb == NULL)
6333                 return (ENOMEM);
6334
6335         t4_read_mtu_tbl(sc, mtus, NULL);
6336
6337         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6338             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6339             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6340             mtus[14], mtus[15]);
6341
6342         rc = sbuf_finish(sb);
6343         sbuf_delete(sb);
6344
6345         return (rc);
6346 }
6347
6348 static int
6349 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6350 {
6351         struct adapter *sc = arg1;
6352         struct sbuf *sb;
6353         int rc, i;
6354         uint32_t cnt[PM_NSTATS];
6355         uint64_t cyc[PM_NSTATS];
6356         static const char *rx_stats[] = {
6357                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6358         };
6359         static const char *tx_stats[] = {
6360                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6361         };
6362
6363         rc = sysctl_wire_old_buffer(req, 0);
6364         if (rc != 0)
6365                 return (rc);
6366
6367         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6368         if (sb == NULL)
6369                 return (ENOMEM);
6370
6371         t4_pmtx_get_stats(sc, cnt, cyc);
6372         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6373         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6374                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6375                     cyc[i]);
6376
6377         t4_pmrx_get_stats(sc, cnt, cyc);
6378         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6379         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6380                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6381                     cyc[i]);
6382
6383         rc = sbuf_finish(sb);
6384         sbuf_delete(sb);
6385
6386         return (rc);
6387 }
6388
6389 static int
6390 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6391 {
6392         struct adapter *sc = arg1;
6393         struct sbuf *sb;
6394         int rc;
6395         struct tp_rdma_stats stats;
6396
6397         rc = sysctl_wire_old_buffer(req, 0);
6398         if (rc != 0)
6399                 return (rc);
6400
6401         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6402         if (sb == NULL)
6403                 return (ENOMEM);
6404
6405         t4_tp_get_rdma_stats(sc, &stats);
6406         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6407         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6408
6409         rc = sbuf_finish(sb);
6410         sbuf_delete(sb);
6411
6412         return (rc);
6413 }
6414
6415 static int
6416 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6417 {
6418         struct adapter *sc = arg1;
6419         struct sbuf *sb;
6420         int rc;
6421         struct tp_tcp_stats v4, v6;
6422
6423         rc = sysctl_wire_old_buffer(req, 0);
6424         if (rc != 0)
6425                 return (rc);
6426
6427         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6428         if (sb == NULL)
6429                 return (ENOMEM);
6430
6431         t4_tp_get_tcp_stats(sc, &v4, &v6);
6432         sbuf_printf(sb,
6433             "                                IP                 IPv6\n");
6434         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6435             v4.tcpOutRsts, v6.tcpOutRsts);
6436         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6437             v4.tcpInSegs, v6.tcpInSegs);
6438         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6439             v4.tcpOutSegs, v6.tcpOutSegs);
6440         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6441             v4.tcpRetransSegs, v6.tcpRetransSegs);
6442
6443         rc = sbuf_finish(sb);
6444         sbuf_delete(sb);
6445
6446         return (rc);
6447 }
6448
6449 static int
6450 sysctl_tids(SYSCTL_HANDLER_ARGS)
6451 {
6452         struct adapter *sc = arg1;
6453         struct sbuf *sb;
6454         int rc;
6455         struct tid_info *t = &sc->tids;
6456
6457         rc = sysctl_wire_old_buffer(req, 0);
6458         if (rc != 0)
6459                 return (rc);
6460
6461         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6462         if (sb == NULL)
6463                 return (ENOMEM);
6464
6465         if (t->natids) {
6466                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6467                     t->atids_in_use);
6468         }
6469
6470         if (t->ntids) {
6471                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6472                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6473
6474                         if (b) {
6475                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6476                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6477                                     t->ntids - 1);
6478                         } else {
6479                                 sbuf_printf(sb, "TID range: %u-%u",
6480                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6481                                     t->ntids - 1);
6482                         }
6483                 } else
6484                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6485                 sbuf_printf(sb, ", in use: %u\n",
6486                     atomic_load_acq_int(&t->tids_in_use));
6487         }
6488
6489         if (t->nstids) {
6490                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6491                     t->stid_base + t->nstids - 1, t->stids_in_use);
6492         }
6493
6494         if (t->nftids) {
6495                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6496                     t->ftid_base + t->nftids - 1);
6497         }
6498
6499         if (t->netids) {
6500                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6501                     t->etid_base + t->netids - 1);
6502         }
6503
6504         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6505             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6506             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6507
6508         rc = sbuf_finish(sb);
6509         sbuf_delete(sb);
6510
6511         return (rc);
6512 }
6513
6514 static int
6515 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6516 {
6517         struct adapter *sc = arg1;
6518         struct sbuf *sb;
6519         int rc;
6520         struct tp_err_stats stats;
6521
6522         rc = sysctl_wire_old_buffer(req, 0);
6523         if (rc != 0)
6524                 return (rc);
6525
6526         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6527         if (sb == NULL)
6528                 return (ENOMEM);
6529
6530         t4_tp_get_err_stats(sc, &stats);
6531
6532         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6533                       "channel 3\n");
6534         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6535             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6536             stats.macInErrs[3]);
6537         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6538             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6539             stats.hdrInErrs[3]);
6540         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6541             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6542             stats.tcpInErrs[3]);
6543         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6544             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6545             stats.tcp6InErrs[3]);
6546         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6547             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6548             stats.tnlCongDrops[3]);
6549         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6550             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6551             stats.tnlTxDrops[3]);
6552         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6553             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6554             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6555         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6556             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6557             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6558         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6559             stats.ofldNoNeigh, stats.ofldCongDefer);
6560
6561         rc = sbuf_finish(sb);
6562         sbuf_delete(sb);
6563
6564         return (rc);
6565 }
6566
6567 struct field_desc {
6568         const char *name;
6569         u_int start;
6570         u_int width;
6571 };
6572
6573 static void
6574 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6575 {
6576         char buf[32];
6577         int line_size = 0;
6578
6579         while (f->name) {
6580                 uint64_t mask = (1ULL << f->width) - 1;
6581                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6582                     ((uintmax_t)v >> f->start) & mask);
6583
6584                 if (line_size + len >= 79) {
6585                         line_size = 8;
6586                         sbuf_printf(sb, "\n        ");
6587                 }
6588                 sbuf_printf(sb, "%s ", buf);
6589                 line_size += len + 1;
6590                 f++;
6591         }
6592         sbuf_printf(sb, "\n");
6593 }
6594
6595 static struct field_desc tp_la0[] = {
6596         { "RcfOpCodeOut", 60, 4 },
6597         { "State", 56, 4 },
6598         { "WcfState", 52, 4 },
6599         { "RcfOpcSrcOut", 50, 2 },
6600         { "CRxError", 49, 1 },
6601         { "ERxError", 48, 1 },
6602         { "SanityFailed", 47, 1 },
6603         { "SpuriousMsg", 46, 1 },
6604         { "FlushInputMsg", 45, 1 },
6605         { "FlushInputCpl", 44, 1 },
6606         { "RssUpBit", 43, 1 },
6607         { "RssFilterHit", 42, 1 },
6608         { "Tid", 32, 10 },
6609         { "InitTcb", 31, 1 },
6610         { "LineNumber", 24, 7 },
6611         { "Emsg", 23, 1 },
6612         { "EdataOut", 22, 1 },
6613         { "Cmsg", 21, 1 },
6614         { "CdataOut", 20, 1 },
6615         { "EreadPdu", 19, 1 },
6616         { "CreadPdu", 18, 1 },
6617         { "TunnelPkt", 17, 1 },
6618         { "RcfPeerFin", 16, 1 },
6619         { "RcfReasonOut", 12, 4 },
6620         { "TxCchannel", 10, 2 },
6621         { "RcfTxChannel", 8, 2 },
6622         { "RxEchannel", 6, 2 },
6623         { "RcfRxChannel", 5, 1 },
6624         { "RcfDataOutSrdy", 4, 1 },
6625         { "RxDvld", 3, 1 },
6626         { "RxOoDvld", 2, 1 },
6627         { "RxCongestion", 1, 1 },
6628         { "TxCongestion", 0, 1 },
6629         { NULL }
6630 };
6631
6632 static struct field_desc tp_la1[] = {
6633         { "CplCmdIn", 56, 8 },
6634         { "CplCmdOut", 48, 8 },
6635         { "ESynOut", 47, 1 },
6636         { "EAckOut", 46, 1 },
6637         { "EFinOut", 45, 1 },
6638         { "ERstOut", 44, 1 },
6639         { "SynIn", 43, 1 },
6640         { "AckIn", 42, 1 },
6641         { "FinIn", 41, 1 },
6642         { "RstIn", 40, 1 },
6643         { "DataIn", 39, 1 },
6644         { "DataInVld", 38, 1 },
6645         { "PadIn", 37, 1 },
6646         { "RxBufEmpty", 36, 1 },
6647         { "RxDdp", 35, 1 },
6648         { "RxFbCongestion", 34, 1 },
6649         { "TxFbCongestion", 33, 1 },
6650         { "TxPktSumSrdy", 32, 1 },
6651         { "RcfUlpType", 28, 4 },
6652         { "Eread", 27, 1 },
6653         { "Ebypass", 26, 1 },
6654         { "Esave", 25, 1 },
6655         { "Static0", 24, 1 },
6656         { "Cread", 23, 1 },
6657         { "Cbypass", 22, 1 },
6658         { "Csave", 21, 1 },
6659         { "CPktOut", 20, 1 },
6660         { "RxPagePoolFull", 18, 2 },
6661         { "RxLpbkPkt", 17, 1 },
6662         { "TxLpbkPkt", 16, 1 },
6663         { "RxVfValid", 15, 1 },
6664         { "SynLearned", 14, 1 },
6665         { "SetDelEntry", 13, 1 },
6666         { "SetInvEntry", 12, 1 },
6667         { "CpcmdDvld", 11, 1 },
6668         { "CpcmdSave", 10, 1 },
6669         { "RxPstructsFull", 8, 2 },
6670         { "EpcmdDvld", 7, 1 },
6671         { "EpcmdFlush", 6, 1 },
6672         { "EpcmdTrimPrefix", 5, 1 },
6673         { "EpcmdTrimPostfix", 4, 1 },
6674         { "ERssIp4Pkt", 3, 1 },
6675         { "ERssIp6Pkt", 2, 1 },
6676         { "ERssTcpUdpPkt", 1, 1 },
6677         { "ERssFceFipPkt", 0, 1 },
6678         { NULL }
6679 };
6680
6681 static struct field_desc tp_la2[] = {
6682         { "CplCmdIn", 56, 8 },
6683         { "MpsVfVld", 55, 1 },
6684         { "MpsPf", 52, 3 },
6685         { "MpsVf", 44, 8 },
6686         { "SynIn", 43, 1 },
6687         { "AckIn", 42, 1 },
6688         { "FinIn", 41, 1 },
6689         { "RstIn", 40, 1 },
6690         { "DataIn", 39, 1 },
6691         { "DataInVld", 38, 1 },
6692         { "PadIn", 37, 1 },
6693         { "RxBufEmpty", 36, 1 },
6694         { "RxDdp", 35, 1 },
6695         { "RxFbCongestion", 34, 1 },
6696         { "TxFbCongestion", 33, 1 },
6697         { "TxPktSumSrdy", 32, 1 },
6698         { "RcfUlpType", 28, 4 },
6699         { "Eread", 27, 1 },
6700         { "Ebypass", 26, 1 },
6701         { "Esave", 25, 1 },
6702         { "Static0", 24, 1 },
6703         { "Cread", 23, 1 },
6704         { "Cbypass", 22, 1 },
6705         { "Csave", 21, 1 },
6706         { "CPktOut", 20, 1 },
6707         { "RxPagePoolFull", 18, 2 },
6708         { "RxLpbkPkt", 17, 1 },
6709         { "TxLpbkPkt", 16, 1 },
6710         { "RxVfValid", 15, 1 },
6711         { "SynLearned", 14, 1 },
6712         { "SetDelEntry", 13, 1 },
6713         { "SetInvEntry", 12, 1 },
6714         { "CpcmdDvld", 11, 1 },
6715         { "CpcmdSave", 10, 1 },
6716         { "RxPstructsFull", 8, 2 },
6717         { "EpcmdDvld", 7, 1 },
6718         { "EpcmdFlush", 6, 1 },
6719         { "EpcmdTrimPrefix", 5, 1 },
6720         { "EpcmdTrimPostfix", 4, 1 },
6721         { "ERssIp4Pkt", 3, 1 },
6722         { "ERssIp6Pkt", 2, 1 },
6723         { "ERssTcpUdpPkt", 1, 1 },
6724         { "ERssFceFipPkt", 0, 1 },
6725         { NULL }
6726 };
6727
6728 static void
6729 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6730 {
6731
6732         field_desc_show(sb, *p, tp_la0);
6733 }
6734
6735 static void
6736 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6737 {
6738
6739         if (idx)
6740                 sbuf_printf(sb, "\n");
6741         field_desc_show(sb, p[0], tp_la0);
6742         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6743                 field_desc_show(sb, p[1], tp_la0);
6744 }
6745
6746 static void
6747 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6748 {
6749
6750         if (idx)
6751                 sbuf_printf(sb, "\n");
6752         field_desc_show(sb, p[0], tp_la0);
6753         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6754                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6755 }
6756
6757 static int
6758 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6759 {
6760         struct adapter *sc = arg1;
6761         struct sbuf *sb;
6762         uint64_t *buf, *p;
6763         int rc;
6764         u_int i, inc;
6765         void (*show_func)(struct sbuf *, uint64_t *, int);
6766
6767         rc = sysctl_wire_old_buffer(req, 0);
6768         if (rc != 0)
6769                 return (rc);
6770
6771         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6772         if (sb == NULL)
6773                 return (ENOMEM);
6774
6775         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6776
6777         t4_tp_read_la(sc, buf, NULL);
6778         p = buf;
6779
6780         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6781         case 2:
6782                 inc = 2;
6783                 show_func = tp_la_show2;
6784                 break;
6785         case 3:
6786                 inc = 2;
6787                 show_func = tp_la_show3;
6788                 break;
6789         default:
6790                 inc = 1;
6791                 show_func = tp_la_show;
6792         }
6793
6794         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6795                 (*show_func)(sb, p, i);
6796
6797         rc = sbuf_finish(sb);
6798         sbuf_delete(sb);
6799         free(buf, M_CXGBE);
6800         return (rc);
6801 }
6802
6803 static int
6804 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6805 {
6806         struct adapter *sc = arg1;
6807         struct sbuf *sb;
6808         int rc;
6809         u64 nrate[NCHAN], orate[NCHAN];
6810
6811         rc = sysctl_wire_old_buffer(req, 0);
6812         if (rc != 0)
6813                 return (rc);
6814
6815         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6816         if (sb == NULL)
6817                 return (ENOMEM);
6818
6819         t4_get_chan_txrate(sc, nrate, orate);
6820         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6821                  "channel 3\n");
6822         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6823             nrate[0], nrate[1], nrate[2], nrate[3]);
6824         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6825             orate[0], orate[1], orate[2], orate[3]);
6826
6827         rc = sbuf_finish(sb);
6828         sbuf_delete(sb);
6829
6830         return (rc);
6831 }
6832
6833 static int
6834 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6835 {
6836         struct adapter *sc = arg1;
6837         struct sbuf *sb;
6838         uint32_t *buf, *p;
6839         int rc, i;
6840
6841         rc = sysctl_wire_old_buffer(req, 0);
6842         if (rc != 0)
6843                 return (rc);
6844
6845         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6846         if (sb == NULL)
6847                 return (ENOMEM);
6848
6849         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6850             M_ZERO | M_WAITOK);
6851
6852         t4_ulprx_read_la(sc, buf);
6853         p = buf;
6854
6855         sbuf_printf(sb, "      Pcmd        Type   Message"
6856             "                Data");
6857         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6858                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6859                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6860         }
6861
6862         rc = sbuf_finish(sb);
6863         sbuf_delete(sb);
6864         free(buf, M_CXGBE);
6865         return (rc);
6866 }
6867
6868 static int
6869 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6870 {
6871         struct adapter *sc = arg1;
6872         struct sbuf *sb;
6873         int rc, v;
6874
6875         rc = sysctl_wire_old_buffer(req, 0);
6876         if (rc != 0)
6877                 return (rc);
6878
6879         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6880         if (sb == NULL)
6881                 return (ENOMEM);
6882
6883         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6884         if (G_STATSOURCE_T5(v) == 7) {
6885                 if (G_STATMODE(v) == 0) {
6886                         sbuf_printf(sb, "total %d, incomplete %d",
6887                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6888                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6889                 } else if (G_STATMODE(v) == 1) {
6890                         sbuf_printf(sb, "total %d, data overflow %d",
6891                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6892                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6893                 }
6894         }
6895         rc = sbuf_finish(sb);
6896         sbuf_delete(sb);
6897
6898         return (rc);
6899 }
6900 #endif
6901
6902 static uint32_t
6903 fconf_to_mode(uint32_t fconf)
6904 {
6905         uint32_t mode;
6906
6907         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6908             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6909
6910         if (fconf & F_FRAGMENTATION)
6911                 mode |= T4_FILTER_IP_FRAGMENT;
6912
6913         if (fconf & F_MPSHITTYPE)
6914                 mode |= T4_FILTER_MPS_HIT_TYPE;
6915
6916         if (fconf & F_MACMATCH)
6917                 mode |= T4_FILTER_MAC_IDX;
6918
6919         if (fconf & F_ETHERTYPE)
6920                 mode |= T4_FILTER_ETH_TYPE;
6921
6922         if (fconf & F_PROTOCOL)
6923                 mode |= T4_FILTER_IP_PROTO;
6924
6925         if (fconf & F_TOS)
6926                 mode |= T4_FILTER_IP_TOS;
6927
6928         if (fconf & F_VLAN)
6929                 mode |= T4_FILTER_VLAN;
6930
6931         if (fconf & F_VNIC_ID)
6932                 mode |= T4_FILTER_VNIC;
6933
6934         if (fconf & F_PORT)
6935                 mode |= T4_FILTER_PORT;
6936
6937         if (fconf & F_FCOE)
6938                 mode |= T4_FILTER_FCoE;
6939
6940         return (mode);
6941 }
6942
6943 static uint32_t
6944 mode_to_fconf(uint32_t mode)
6945 {
6946         uint32_t fconf = 0;
6947
6948         if (mode & T4_FILTER_IP_FRAGMENT)
6949                 fconf |= F_FRAGMENTATION;
6950
6951         if (mode & T4_FILTER_MPS_HIT_TYPE)
6952                 fconf |= F_MPSHITTYPE;
6953
6954         if (mode & T4_FILTER_MAC_IDX)
6955                 fconf |= F_MACMATCH;
6956
6957         if (mode & T4_FILTER_ETH_TYPE)
6958                 fconf |= F_ETHERTYPE;
6959
6960         if (mode & T4_FILTER_IP_PROTO)
6961                 fconf |= F_PROTOCOL;
6962
6963         if (mode & T4_FILTER_IP_TOS)
6964                 fconf |= F_TOS;
6965
6966         if (mode & T4_FILTER_VLAN)
6967                 fconf |= F_VLAN;
6968
6969         if (mode & T4_FILTER_VNIC)
6970                 fconf |= F_VNIC_ID;
6971
6972         if (mode & T4_FILTER_PORT)
6973                 fconf |= F_PORT;
6974
6975         if (mode & T4_FILTER_FCoE)
6976                 fconf |= F_FCOE;
6977
6978         return (fconf);
6979 }
6980
6981 static uint32_t
6982 fspec_to_fconf(struct t4_filter_specification *fs)
6983 {
6984         uint32_t fconf = 0;
6985
6986         if (fs->val.frag || fs->mask.frag)
6987                 fconf |= F_FRAGMENTATION;
6988
6989         if (fs->val.matchtype || fs->mask.matchtype)
6990                 fconf |= F_MPSHITTYPE;
6991
6992         if (fs->val.macidx || fs->mask.macidx)
6993                 fconf |= F_MACMATCH;
6994
6995         if (fs->val.ethtype || fs->mask.ethtype)
6996                 fconf |= F_ETHERTYPE;
6997
6998         if (fs->val.proto || fs->mask.proto)
6999                 fconf |= F_PROTOCOL;
7000
7001         if (fs->val.tos || fs->mask.tos)
7002                 fconf |= F_TOS;
7003
7004         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7005                 fconf |= F_VLAN;
7006
7007         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7008                 fconf |= F_VNIC_ID;
7009
7010         if (fs->val.iport || fs->mask.iport)
7011                 fconf |= F_PORT;
7012
7013         if (fs->val.fcoe || fs->mask.fcoe)
7014                 fconf |= F_FCOE;
7015
7016         return (fconf);
7017 }
7018
7019 static int
7020 get_filter_mode(struct adapter *sc, uint32_t *mode)
7021 {
7022         int rc;
7023         uint32_t fconf;
7024
7025         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7026             "t4getfm");
7027         if (rc)
7028                 return (rc);
7029
7030         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7031             A_TP_VLAN_PRI_MAP);
7032
7033         if (sc->params.tp.vlan_pri_map != fconf) {
7034                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7035                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7036                     fconf);
7037         }
7038
7039         *mode = fconf_to_mode(fconf);
7040
7041         end_synchronized_op(sc, LOCK_HELD);
7042         return (0);
7043 }
7044
7045 static int
7046 set_filter_mode(struct adapter *sc, uint32_t mode)
7047 {
7048         uint32_t fconf;
7049         int rc;
7050
7051         fconf = mode_to_fconf(mode);
7052
7053         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7054             "t4setfm");
7055         if (rc)
7056                 return (rc);
7057
7058         if (sc->tids.ftids_in_use > 0) {
7059                 rc = EBUSY;
7060                 goto done;
7061         }
7062
7063 #ifdef TCP_OFFLOAD
7064         if (uld_active(sc, ULD_TOM)) {
7065                 rc = EBUSY;
7066                 goto done;
7067         }
7068 #endif
7069
7070         rc = -t4_set_filter_mode(sc, fconf);
7071 done:
7072         end_synchronized_op(sc, LOCK_HELD);
7073         return (rc);
7074 }
7075
7076 static inline uint64_t
7077 get_filter_hits(struct adapter *sc, uint32_t fid)
7078 {
7079         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7080         uint64_t hits;
7081
7082         memwin_info(sc, 0, &mw_base, NULL);
7083         off = position_memwin(sc, 0,
7084             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7085         if (is_t4(sc)) {
7086                 hits = t4_read_reg64(sc, mw_base + off + 16);
7087                 hits = be64toh(hits);
7088         } else {
7089                 hits = t4_read_reg(sc, mw_base + off + 24);
7090                 hits = be32toh(hits);
7091         }
7092
7093         return (hits);
7094 }
7095
7096 static int
7097 get_filter(struct adapter *sc, struct t4_filter *t)
7098 {
7099         int i, rc, nfilters = sc->tids.nftids;
7100         struct filter_entry *f;
7101
7102         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7103             "t4getf");
7104         if (rc)
7105                 return (rc);
7106
7107         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7108             t->idx >= nfilters) {
7109                 t->idx = 0xffffffff;
7110                 goto done;
7111         }
7112
7113         f = &sc->tids.ftid_tab[t->idx];
7114         for (i = t->idx; i < nfilters; i++, f++) {
7115                 if (f->valid) {
7116                         t->idx = i;
7117                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7118                         t->smtidx = f->smtidx;
7119                         if (f->fs.hitcnts)
7120                                 t->hits = get_filter_hits(sc, t->idx);
7121                         else
7122                                 t->hits = UINT64_MAX;
7123                         t->fs = f->fs;
7124
7125                         goto done;
7126                 }
7127         }
7128
7129         t->idx = 0xffffffff;
7130 done:
7131         end_synchronized_op(sc, LOCK_HELD);
7132         return (0);
7133 }
7134
7135 static int
7136 set_filter(struct adapter *sc, struct t4_filter *t)
7137 {
7138         unsigned int nfilters, nports;
7139         struct filter_entry *f;
7140         int i, rc;
7141
7142         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7143         if (rc)
7144                 return (rc);
7145
7146         nfilters = sc->tids.nftids;
7147         nports = sc->params.nports;
7148
7149         if (nfilters == 0) {
7150                 rc = ENOTSUP;
7151                 goto done;
7152         }
7153
7154         if (!(sc->flags & FULL_INIT_DONE)) {
7155                 rc = EAGAIN;
7156                 goto done;
7157         }
7158
7159         if (t->idx >= nfilters) {
7160                 rc = EINVAL;
7161                 goto done;
7162         }
7163
7164         /* Validate against the global filter mode */
7165         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7166             sc->params.tp.vlan_pri_map) {
7167                 rc = E2BIG;
7168                 goto done;
7169         }
7170
7171         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7172                 rc = EINVAL;
7173                 goto done;
7174         }
7175
7176         if (t->fs.val.iport >= nports) {
7177                 rc = EINVAL;
7178                 goto done;
7179         }
7180
7181         /* Can't specify an iq if not steering to it */
7182         if (!t->fs.dirsteer && t->fs.iq) {
7183                 rc = EINVAL;
7184                 goto done;
7185         }
7186
7187         /* IPv6 filter idx must be 4 aligned */
7188         if (t->fs.type == 1 &&
7189             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7190                 rc = EINVAL;
7191                 goto done;
7192         }
7193
7194         if (sc->tids.ftid_tab == NULL) {
7195                 KASSERT(sc->tids.ftids_in_use == 0,
7196                     ("%s: no memory allocated but filters_in_use > 0",
7197                     __func__));
7198
7199                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7200                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7201                 if (sc->tids.ftid_tab == NULL) {
7202                         rc = ENOMEM;
7203                         goto done;
7204                 }
7205                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7206         }
7207
7208         for (i = 0; i < 4; i++) {
7209                 f = &sc->tids.ftid_tab[t->idx + i];
7210
7211                 if (f->pending || f->valid) {
7212                         rc = EBUSY;
7213                         goto done;
7214                 }
7215                 if (f->locked) {
7216                         rc = EPERM;
7217                         goto done;
7218                 }
7219
7220                 if (t->fs.type == 0)
7221                         break;
7222         }
7223
7224         f = &sc->tids.ftid_tab[t->idx];
7225         f->fs = t->fs;
7226
7227         rc = set_filter_wr(sc, t->idx);
7228 done:
7229         end_synchronized_op(sc, 0);
7230
7231         if (rc == 0) {
7232                 mtx_lock(&sc->tids.ftid_lock);
7233                 for (;;) {
7234                         if (f->pending == 0) {
7235                                 rc = f->valid ? 0 : EIO;
7236                                 break;
7237                         }
7238
7239                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7240                             PCATCH, "t4setfw", 0)) {
7241                                 rc = EINPROGRESS;
7242                                 break;
7243                         }
7244                 }
7245                 mtx_unlock(&sc->tids.ftid_lock);
7246         }
7247         return (rc);
7248 }
7249
7250 static int
7251 del_filter(struct adapter *sc, struct t4_filter *t)
7252 {
7253         unsigned int nfilters;
7254         struct filter_entry *f;
7255         int rc;
7256
7257         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7258         if (rc)
7259                 return (rc);
7260
7261         nfilters = sc->tids.nftids;
7262
7263         if (nfilters == 0) {
7264                 rc = ENOTSUP;
7265                 goto done;
7266         }
7267
7268         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7269             t->idx >= nfilters) {
7270                 rc = EINVAL;
7271                 goto done;
7272         }
7273
7274         if (!(sc->flags & FULL_INIT_DONE)) {
7275                 rc = EAGAIN;
7276                 goto done;
7277         }
7278
7279         f = &sc->tids.ftid_tab[t->idx];
7280
7281         if (f->pending) {
7282                 rc = EBUSY;
7283                 goto done;
7284         }
7285         if (f->locked) {
7286                 rc = EPERM;
7287                 goto done;
7288         }
7289
7290         if (f->valid) {
7291                 t->fs = f->fs;  /* extra info for the caller */
7292                 rc = del_filter_wr(sc, t->idx);
7293         }
7294
7295 done:
7296         end_synchronized_op(sc, 0);
7297
7298         if (rc == 0) {
7299                 mtx_lock(&sc->tids.ftid_lock);
7300                 for (;;) {
7301                         if (f->pending == 0) {
7302                                 rc = f->valid ? EIO : 0;
7303                                 break;
7304                         }
7305
7306                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7307                             PCATCH, "t4delfw", 0)) {
7308                                 rc = EINPROGRESS;
7309                                 break;
7310                         }
7311                 }
7312                 mtx_unlock(&sc->tids.ftid_lock);
7313         }
7314
7315         return (rc);
7316 }
7317
7318 static void
7319 clear_filter(struct filter_entry *f)
7320 {
7321         if (f->l2t)
7322                 t4_l2t_release(f->l2t);
7323
7324         bzero(f, sizeof (*f));
7325 }
7326
7327 static int
7328 set_filter_wr(struct adapter *sc, int fidx)
7329 {
7330         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7331         struct fw_filter_wr *fwr;
7332         unsigned int ftid;
7333         struct wrq_cookie cookie;
7334
7335         ASSERT_SYNCHRONIZED_OP(sc);
7336
7337         if (f->fs.newdmac || f->fs.newvlan) {
7338                 /* This filter needs an L2T entry; allocate one. */
7339                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7340                 if (f->l2t == NULL)
7341                         return (EAGAIN);
7342                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7343                     f->fs.dmac)) {
7344                         t4_l2t_release(f->l2t);
7345                         f->l2t = NULL;
7346                         return (ENOMEM);
7347                 }
7348         }
7349
7350         ftid = sc->tids.ftid_base + fidx;
7351
7352         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7353         if (fwr == NULL)
7354                 return (ENOMEM);
7355         bzero(fwr, sizeof(*fwr));
7356
7357         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7358         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7359         fwr->tid_to_iq =
7360             htobe32(V_FW_FILTER_WR_TID(ftid) |
7361                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7362                 V_FW_FILTER_WR_NOREPLY(0) |
7363                 V_FW_FILTER_WR_IQ(f->fs.iq));
7364         fwr->del_filter_to_l2tix =
7365             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7366                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7367                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7368                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7369                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7370                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7371                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7372                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7373                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7374                     f->fs.newvlan == VLAN_REWRITE) |
7375                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7376                     f->fs.newvlan == VLAN_REWRITE) |
7377                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7378                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7379                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7380                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7381         fwr->ethtype = htobe16(f->fs.val.ethtype);
7382         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7383         fwr->frag_to_ovlan_vldm =
7384             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7385                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7386                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7387                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7388                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7389                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7390         fwr->smac_sel = 0;
7391         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7392             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7393         fwr->maci_to_matchtypem =
7394             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7395                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7396                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7397                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7398                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7399                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7400                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7401                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7402         fwr->ptcl = f->fs.val.proto;
7403         fwr->ptclm = f->fs.mask.proto;
7404         fwr->ttyp = f->fs.val.tos;
7405         fwr->ttypm = f->fs.mask.tos;
7406         fwr->ivlan = htobe16(f->fs.val.vlan);
7407         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7408         fwr->ovlan = htobe16(f->fs.val.vnic);
7409         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7410         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7411         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7412         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7413         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7414         fwr->lp = htobe16(f->fs.val.dport);
7415         fwr->lpm = htobe16(f->fs.mask.dport);
7416         fwr->fp = htobe16(f->fs.val.sport);
7417         fwr->fpm = htobe16(f->fs.mask.sport);
7418         if (f->fs.newsmac)
7419                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7420
7421         f->pending = 1;
7422         sc->tids.ftids_in_use++;
7423
7424         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7425         return (0);
7426 }
7427
7428 static int
7429 del_filter_wr(struct adapter *sc, int fidx)
7430 {
7431         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7432         struct fw_filter_wr *fwr;
7433         unsigned int ftid;
7434         struct wrq_cookie cookie;
7435
7436         ftid = sc->tids.ftid_base + fidx;
7437
7438         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7439         if (fwr == NULL)
7440                 return (ENOMEM);
7441         bzero(fwr, sizeof (*fwr));
7442
7443         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7444
7445         f->pending = 1;
7446         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7447         return (0);
7448 }
7449
7450 int
7451 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7452 {
7453         struct adapter *sc = iq->adapter;
7454         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7455         unsigned int idx = GET_TID(rpl);
7456         unsigned int rc;
7457         struct filter_entry *f;
7458
7459         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7460             rss->opcode));
7461
7462         if (is_ftid(sc, idx)) {
7463
7464                 idx -= sc->tids.ftid_base;
7465                 f = &sc->tids.ftid_tab[idx];
7466                 rc = G_COOKIE(rpl->cookie);
7467
7468                 mtx_lock(&sc->tids.ftid_lock);
7469                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7470                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7471                             __func__, idx));
7472                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7473                         f->pending = 0;  /* asynchronous setup completed */
7474                         f->valid = 1;
7475                 } else {
7476                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7477                                 /* Add or delete failed, display an error */
7478                                 log(LOG_ERR,
7479                                     "filter %u setup failed with error %u\n",
7480                                     idx, rc);
7481                         }
7482
7483                         clear_filter(f);
7484                         sc->tids.ftids_in_use--;
7485                 }
7486                 wakeup(&sc->tids.ftid_tab);
7487                 mtx_unlock(&sc->tids.ftid_lock);
7488         }
7489
7490         return (0);
7491 }
7492
7493 static int
7494 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7495 {
7496         int rc;
7497
7498         if (cntxt->cid > M_CTXTQID)
7499                 return (EINVAL);
7500
7501         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7502             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7503                 return (EINVAL);
7504
7505         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7506         if (rc)
7507                 return (rc);
7508
7509         if (sc->flags & FW_OK) {
7510                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7511                     &cntxt->data[0]);
7512                 if (rc == 0)
7513                         goto done;
7514         }
7515
7516         /*
7517          * Read via firmware failed or wasn't even attempted.  Read directly via
7518          * the backdoor.
7519          */
7520         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7521 done:
7522         end_synchronized_op(sc, 0);
7523         return (rc);
7524 }
7525
7526 static int
7527 load_fw(struct adapter *sc, struct t4_data *fw)
7528 {
7529         int rc;
7530         uint8_t *fw_data;
7531
7532         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7533         if (rc)
7534                 return (rc);
7535
7536         if (sc->flags & FULL_INIT_DONE) {
7537                 rc = EBUSY;
7538                 goto done;
7539         }
7540
7541         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7542         if (fw_data == NULL) {
7543                 rc = ENOMEM;
7544                 goto done;
7545         }
7546
7547         rc = copyin(fw->data, fw_data, fw->len);
7548         if (rc == 0)
7549                 rc = -t4_load_fw(sc, fw_data, fw->len);
7550
7551         free(fw_data, M_CXGBE);
7552 done:
7553         end_synchronized_op(sc, 0);
7554         return (rc);
7555 }
7556
7557 static int
7558 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7559 {
7560         uint32_t addr, off, remaining, i, n;
7561         uint32_t *buf, *b;
7562         uint32_t mw_base, mw_aperture;
7563         int rc;
7564         uint8_t *dst;
7565
7566         rc = validate_mem_range(sc, mr->addr, mr->len);
7567         if (rc != 0)
7568                 return (rc);
7569
7570         memwin_info(sc, win, &mw_base, &mw_aperture);
7571         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7572         addr = mr->addr;
7573         remaining = mr->len;
7574         dst = (void *)mr->data;
7575
7576         while (remaining) {
7577                 off = position_memwin(sc, win, addr);
7578
7579                 /* number of bytes that we'll copy in the inner loop */
7580                 n = min(remaining, mw_aperture - off);
7581                 for (i = 0; i < n; i += 4)
7582                         *b++ = t4_read_reg(sc, mw_base + off + i);
7583
7584                 rc = copyout(buf, dst, n);
7585                 if (rc != 0)
7586                         break;
7587
7588                 b = buf;
7589                 dst += n;
7590                 remaining -= n;
7591                 addr += n;
7592         }
7593
7594         free(buf, M_CXGBE);
7595         return (rc);
7596 }
7597
7598 static int
7599 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7600 {
7601         int rc;
7602
7603         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7604                 return (EINVAL);
7605
7606         if (i2cd->len > sizeof(i2cd->data))
7607                 return (EFBIG);
7608
7609         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7610         if (rc)
7611                 return (rc);
7612         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7613             i2cd->offset, i2cd->len, &i2cd->data[0]);
7614         end_synchronized_op(sc, 0);
7615
7616         return (rc);
7617 }
7618
7619 static int
7620 in_range(int val, int lo, int hi)
7621 {
7622
7623         return (val < 0 || (val <= hi && val >= lo));
7624 }
7625
7626 static int
7627 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7628 {
7629         int fw_subcmd, fw_type, rc;
7630
7631         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7632         if (rc)
7633                 return (rc);
7634
7635         if (!(sc->flags & FULL_INIT_DONE)) {
7636                 rc = EAGAIN;
7637                 goto done;
7638         }
7639
7640         /*
7641          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7642          * sub-command and type are in common locations.)
7643          */
7644         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7645                 fw_subcmd = FW_SCHED_SC_CONFIG;
7646         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7647                 fw_subcmd = FW_SCHED_SC_PARAMS;
7648         else {
7649                 rc = EINVAL;
7650                 goto done;
7651         }
7652         if (p->type == SCHED_CLASS_TYPE_PACKET)
7653                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7654         else {
7655                 rc = EINVAL;
7656                 goto done;
7657         }
7658
7659         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7660                 /* Vet our parameters ..*/
7661                 if (p->u.config.minmax < 0) {
7662                         rc = EINVAL;
7663                         goto done;
7664                 }
7665
7666                 /* And pass the request to the firmware ...*/
7667                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7668                 goto done;
7669         }
7670
7671         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7672                 int fw_level;
7673                 int fw_mode;
7674                 int fw_rateunit;
7675                 int fw_ratemode;
7676
7677                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7678                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7679                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7680                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7681                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7682                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7683                 else {
7684                         rc = EINVAL;
7685                         goto done;
7686                 }
7687
7688                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7689                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7690                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7691                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7692                 else {
7693                         rc = EINVAL;
7694                         goto done;
7695                 }
7696
7697                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7698                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7699                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7700                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7701                 else {
7702                         rc = EINVAL;
7703                         goto done;
7704                 }
7705
7706                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7707                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7708                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7709                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7710                 else {
7711                         rc = EINVAL;
7712                         goto done;
7713                 }
7714
7715                 /* Vet our parameters ... */
7716                 if (!in_range(p->u.params.channel, 0, 3) ||
7717                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7718                     !in_range(p->u.params.minrate, 0, 10000000) ||
7719                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7720                     !in_range(p->u.params.weight, 0, 100)) {
7721                         rc = ERANGE;
7722                         goto done;
7723                 }
7724
7725                 /*
7726                  * Translate any unset parameters into the firmware's
7727                  * nomenclature and/or fail the call if the parameters
7728                  * are required ...
7729                  */
7730                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7731                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7732                         rc = EINVAL;
7733                         goto done;
7734                 }
7735                 if (p->u.params.minrate < 0)
7736                         p->u.params.minrate = 0;
7737                 if (p->u.params.maxrate < 0) {
7738                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7739                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7740                                 rc = EINVAL;
7741                                 goto done;
7742                         } else
7743                                 p->u.params.maxrate = 0;
7744                 }
7745                 if (p->u.params.weight < 0) {
7746                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7747                                 rc = EINVAL;
7748                                 goto done;
7749                         } else
7750                                 p->u.params.weight = 0;
7751                 }
7752                 if (p->u.params.pktsize < 0) {
7753                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7754                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7755                                 rc = EINVAL;
7756                                 goto done;
7757                         } else
7758                                 p->u.params.pktsize = 0;
7759                 }
7760
7761                 /* See what the firmware thinks of the request ... */
7762                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7763                     fw_rateunit, fw_ratemode, p->u.params.channel,
7764                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7765                     p->u.params.weight, p->u.params.pktsize, 1);
7766                 goto done;
7767         }
7768
7769         rc = EINVAL;
7770 done:
7771         end_synchronized_op(sc, 0);
7772         return (rc);
7773 }
7774
7775 static int
7776 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7777 {
7778         struct port_info *pi = NULL;
7779         struct sge_txq *txq;
7780         uint32_t fw_mnem, fw_queue, fw_class;
7781         int i, rc;
7782
7783         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7784         if (rc)
7785                 return (rc);
7786
7787         if (!(sc->flags & FULL_INIT_DONE)) {
7788                 rc = EAGAIN;
7789                 goto done;
7790         }
7791
7792         if (p->port >= sc->params.nports) {
7793                 rc = EINVAL;
7794                 goto done;
7795         }
7796
7797         pi = sc->port[p->port];
7798         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7799                 rc = EINVAL;
7800                 goto done;
7801         }
7802
7803         /*
7804          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7805          * Scheduling Class in this case).
7806          */
7807         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7808             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7809         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7810
7811         /*
7812          * If op.queue is non-negative, then we're only changing the scheduling
7813          * on a single specified TX queue.
7814          */
7815         if (p->queue >= 0) {
7816                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7817                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7818                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7819                     &fw_class);
7820                 goto done;
7821         }
7822
7823         /*
7824          * Change the scheduling on all the TX queues for the
7825          * interface.
7826          */
7827         for_each_txq(pi, i, txq) {
7828                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7829                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7830                     &fw_class);
7831                 if (rc)
7832                         goto done;
7833         }
7834
7835         rc = 0;
7836 done:
7837         end_synchronized_op(sc, 0);
7838         return (rc);
7839 }
7840
7841 int
7842 t4_os_find_pci_capability(struct adapter *sc, int cap)
7843 {
7844         int i;
7845
7846         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7847 }
7848
7849 int
7850 t4_os_pci_save_state(struct adapter *sc)
7851 {
7852         device_t dev;
7853         struct pci_devinfo *dinfo;
7854
7855         dev = sc->dev;
7856         dinfo = device_get_ivars(dev);
7857
7858         pci_cfg_save(dev, dinfo, 0);
7859         return (0);
7860 }
7861
7862 int
7863 t4_os_pci_restore_state(struct adapter *sc)
7864 {
7865         device_t dev;
7866         struct pci_devinfo *dinfo;
7867
7868         dev = sc->dev;
7869         dinfo = device_get_ivars(dev);
7870
7871         pci_cfg_restore(dev, dinfo);
7872         return (0);
7873 }
7874
7875 void
7876 t4_os_portmod_changed(const struct adapter *sc, int idx)
7877 {
7878         struct port_info *pi = sc->port[idx];
7879         static const char *mod_str[] = {
7880                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7881         };
7882
7883         build_medialist(pi, &pi->media);
7884 #ifdef DEV_NETMAP
7885         build_medialist(pi, &pi->nm_media);
7886 #endif
7887
7888         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7889                 if_printf(pi->ifp, "transceiver unplugged.\n");
7890         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7891                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7892         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7893                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7894         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7895                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7896                     mod_str[pi->mod_type]);
7897         } else {
7898                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7899                     pi->mod_type);
7900         }
7901 }
7902
7903 void
7904 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7905 {
7906         struct port_info *pi = sc->port[idx];
7907         struct ifnet *ifp = pi->ifp;
7908
7909         if (link_stat) {
7910                 pi->linkdnrc = -1;
7911                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7912                 if_link_state_change(ifp, LINK_STATE_UP);
7913         } else {
7914                 if (reason >= 0)
7915                         pi->linkdnrc = reason;
7916                 if_link_state_change(ifp, LINK_STATE_DOWN);
7917         }
7918 }
7919
7920 void
7921 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7922 {
7923         struct adapter *sc;
7924
7925         sx_slock(&t4_list_lock);
7926         SLIST_FOREACH(sc, &t4_list, link) {
7927                 /*
7928                  * func should not make any assumptions about what state sc is
7929                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7930                  */
7931                 func(sc, arg);
7932         }
7933         sx_sunlock(&t4_list_lock);
7934 }
7935
7936 static int
7937 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7938 {
7939        return (0);
7940 }
7941
7942 static int
7943 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7944 {
7945        return (0);
7946 }
7947
7948 static int
7949 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7950     struct thread *td)
7951 {
7952         int rc;
7953         struct adapter *sc = dev->si_drv1;
7954
7955         rc = priv_check(td, PRIV_DRIVER);
7956         if (rc != 0)
7957                 return (rc);
7958
7959         switch (cmd) {
7960         case CHELSIO_T4_GETREG: {
7961                 struct t4_reg *edata = (struct t4_reg *)data;
7962
7963                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7964                         return (EFAULT);
7965
7966                 if (edata->size == 4)
7967                         edata->val = t4_read_reg(sc, edata->addr);
7968                 else if (edata->size == 8)
7969                         edata->val = t4_read_reg64(sc, edata->addr);
7970                 else
7971                         return (EINVAL);
7972
7973                 break;
7974         }
7975         case CHELSIO_T4_SETREG: {
7976                 struct t4_reg *edata = (struct t4_reg *)data;
7977
7978                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7979                         return (EFAULT);
7980
7981                 if (edata->size == 4) {
7982                         if (edata->val & 0xffffffff00000000)
7983                                 return (EINVAL);
7984                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7985                 } else if (edata->size == 8)
7986                         t4_write_reg64(sc, edata->addr, edata->val);
7987                 else
7988                         return (EINVAL);
7989                 break;
7990         }
7991         case CHELSIO_T4_REGDUMP: {
7992                 struct t4_regdump *regs = (struct t4_regdump *)data;
7993                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7994                 uint8_t *buf;
7995
7996                 if (regs->len < reglen) {
7997                         regs->len = reglen; /* hint to the caller */
7998                         return (ENOBUFS);
7999                 }
8000
8001                 regs->len = reglen;
8002                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8003                 t4_get_regs(sc, regs, buf);
8004                 rc = copyout(buf, regs->data, reglen);
8005                 free(buf, M_CXGBE);
8006                 break;
8007         }
8008         case CHELSIO_T4_GET_FILTER_MODE:
8009                 rc = get_filter_mode(sc, (uint32_t *)data);
8010                 break;
8011         case CHELSIO_T4_SET_FILTER_MODE:
8012                 rc = set_filter_mode(sc, *(uint32_t *)data);
8013                 break;
8014         case CHELSIO_T4_GET_FILTER:
8015                 rc = get_filter(sc, (struct t4_filter *)data);
8016                 break;
8017         case CHELSIO_T4_SET_FILTER:
8018                 rc = set_filter(sc, (struct t4_filter *)data);
8019                 break;
8020         case CHELSIO_T4_DEL_FILTER:
8021                 rc = del_filter(sc, (struct t4_filter *)data);
8022                 break;
8023         case CHELSIO_T4_GET_SGE_CONTEXT:
8024                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8025                 break;
8026         case CHELSIO_T4_LOAD_FW:
8027                 rc = load_fw(sc, (struct t4_data *)data);
8028                 break;
8029         case CHELSIO_T4_GET_MEM:
8030                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8031                 break;
8032         case CHELSIO_T4_GET_I2C:
8033                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8034                 break;
8035         case CHELSIO_T4_CLEAR_STATS: {
8036                 int i;
8037                 u_int port_id = *(uint32_t *)data;
8038                 struct port_info *pi;
8039
8040                 if (port_id >= sc->params.nports)
8041                         return (EINVAL);
8042                 pi = sc->port[port_id];
8043
8044                 /* MAC stats */
8045                 t4_clr_port_stats(sc, pi->tx_chan);
8046                 pi->tx_parse_error = 0;
8047
8048                 if (pi->flags & PORT_INIT_DONE) {
8049                         struct sge_rxq *rxq;
8050                         struct sge_txq *txq;
8051                         struct sge_wrq *wrq;
8052
8053                         for_each_rxq(pi, i, rxq) {
8054 #if defined(INET) || defined(INET6)
8055                                 rxq->lro.lro_queued = 0;
8056                                 rxq->lro.lro_flushed = 0;
8057 #endif
8058                                 rxq->rxcsum = 0;
8059                                 rxq->vlan_extraction = 0;
8060                         }
8061
8062                         for_each_txq(pi, i, txq) {
8063                                 txq->txcsum = 0;
8064                                 txq->tso_wrs = 0;
8065                                 txq->vlan_insertion = 0;
8066                                 txq->imm_wrs = 0;
8067                                 txq->sgl_wrs = 0;
8068                                 txq->txpkt_wrs = 0;
8069                                 txq->txpkts0_wrs = 0;
8070                                 txq->txpkts1_wrs = 0;
8071                                 txq->txpkts0_pkts = 0;
8072                                 txq->txpkts1_pkts = 0;
8073                                 mp_ring_reset_stats(txq->r);
8074                         }
8075
8076 #ifdef TCP_OFFLOAD
8077                         /* nothing to clear for each ofld_rxq */
8078
8079                         for_each_ofld_txq(pi, i, wrq) {
8080                                 wrq->tx_wrs_direct = 0;
8081                                 wrq->tx_wrs_copied = 0;
8082                         }
8083 #endif
8084                         wrq = &sc->sge.ctrlq[pi->port_id];
8085                         wrq->tx_wrs_direct = 0;
8086                         wrq->tx_wrs_copied = 0;
8087                 }
8088                 break;
8089         }
8090         case CHELSIO_T4_SCHED_CLASS:
8091                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8092                 break;
8093         case CHELSIO_T4_SCHED_QUEUE:
8094                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8095                 break;
8096         case CHELSIO_T4_GET_TRACER:
8097                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8098                 break;
8099         case CHELSIO_T4_SET_TRACER:
8100                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8101                 break;
8102         default:
8103                 rc = EINVAL;
8104         }
8105
8106         return (rc);
8107 }
8108
8109 #ifdef TCP_OFFLOAD
8110 void
8111 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8112     const unsigned int *pgsz_order)
8113 {
8114         struct port_info *pi = ifp->if_softc;
8115         struct adapter *sc = pi->adapter;
8116
8117         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8118         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8119                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8120                 V_HPZ3(pgsz_order[3]));
8121 }
8122
8123 static int
8124 toe_capability(struct port_info *pi, int enable)
8125 {
8126         int rc;
8127         struct adapter *sc = pi->adapter;
8128
8129         ASSERT_SYNCHRONIZED_OP(sc);
8130
8131         if (!is_offload(sc))
8132                 return (ENODEV);
8133
8134         if (enable) {
8135                 /*
8136                  * We need the port's queues around so that we're able to send
8137                  * and receive CPLs to/from the TOE even if the ifnet for this
8138                  * port has never been UP'd administratively.
8139                  */
8140                 if (!(pi->flags & PORT_INIT_DONE)) {
8141                         rc = cxgbe_init_synchronized(pi);
8142                         if (rc)
8143                                 return (rc);
8144                 }
8145
8146                 if (isset(&sc->offload_map, pi->port_id))
8147                         return (0);
8148
8149                 if (!uld_active(sc, ULD_TOM)) {
8150                         rc = t4_activate_uld(sc, ULD_TOM);
8151                         if (rc == EAGAIN) {
8152                                 log(LOG_WARNING,
8153                                     "You must kldload t4_tom.ko before trying "
8154                                     "to enable TOE on a cxgbe interface.\n");
8155                         }
8156                         if (rc != 0)
8157                                 return (rc);
8158                         KASSERT(sc->tom_softc != NULL,
8159                             ("%s: TOM activated but softc NULL", __func__));
8160                         KASSERT(uld_active(sc, ULD_TOM),
8161                             ("%s: TOM activated but flag not set", __func__));
8162                 }
8163
8164                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8165                 if (!uld_active(sc, ULD_IWARP))
8166                         (void) t4_activate_uld(sc, ULD_IWARP);
8167                 if (!uld_active(sc, ULD_ISCSI))
8168                         (void) t4_activate_uld(sc, ULD_ISCSI);
8169
8170                 setbit(&sc->offload_map, pi->port_id);
8171         } else {
8172                 if (!isset(&sc->offload_map, pi->port_id))
8173                         return (0);
8174
8175                 KASSERT(uld_active(sc, ULD_TOM),
8176                     ("%s: TOM never initialized?", __func__));
8177                 clrbit(&sc->offload_map, pi->port_id);
8178         }
8179
8180         return (0);
8181 }
8182
8183 /*
8184  * Add an upper layer driver to the global list.
8185  */
8186 int
8187 t4_register_uld(struct uld_info *ui)
8188 {
8189         int rc = 0;
8190         struct uld_info *u;
8191
8192         sx_xlock(&t4_uld_list_lock);
8193         SLIST_FOREACH(u, &t4_uld_list, link) {
8194             if (u->uld_id == ui->uld_id) {
8195                     rc = EEXIST;
8196                     goto done;
8197             }
8198         }
8199
8200         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8201         ui->refcount = 0;
8202 done:
8203         sx_xunlock(&t4_uld_list_lock);
8204         return (rc);
8205 }
8206
8207 int
8208 t4_unregister_uld(struct uld_info *ui)
8209 {
8210         int rc = EINVAL;
8211         struct uld_info *u;
8212
8213         sx_xlock(&t4_uld_list_lock);
8214
8215         SLIST_FOREACH(u, &t4_uld_list, link) {
8216             if (u == ui) {
8217                     if (ui->refcount > 0) {
8218                             rc = EBUSY;
8219                             goto done;
8220                     }
8221
8222                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8223                     rc = 0;
8224                     goto done;
8225             }
8226         }
8227 done:
8228         sx_xunlock(&t4_uld_list_lock);
8229         return (rc);
8230 }
8231
8232 int
8233 t4_activate_uld(struct adapter *sc, int id)
8234 {
8235         int rc;
8236         struct uld_info *ui;
8237
8238         ASSERT_SYNCHRONIZED_OP(sc);
8239
8240         if (id < 0 || id > ULD_MAX)
8241                 return (EINVAL);
8242         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8243
8244         sx_slock(&t4_uld_list_lock);
8245
8246         SLIST_FOREACH(ui, &t4_uld_list, link) {
8247                 if (ui->uld_id == id) {
8248                         if (!(sc->flags & FULL_INIT_DONE)) {
8249                                 rc = adapter_full_init(sc);
8250                                 if (rc != 0)
8251                                         break;
8252                         }
8253
8254                         rc = ui->activate(sc);
8255                         if (rc == 0) {
8256                                 setbit(&sc->active_ulds, id);
8257                                 ui->refcount++;
8258                         }
8259                         break;
8260                 }
8261         }
8262
8263         sx_sunlock(&t4_uld_list_lock);
8264
8265         return (rc);
8266 }
8267
8268 int
8269 t4_deactivate_uld(struct adapter *sc, int id)
8270 {
8271         int rc;
8272         struct uld_info *ui;
8273
8274         ASSERT_SYNCHRONIZED_OP(sc);
8275
8276         if (id < 0 || id > ULD_MAX)
8277                 return (EINVAL);
8278         rc = ENXIO;
8279
8280         sx_slock(&t4_uld_list_lock);
8281
8282         SLIST_FOREACH(ui, &t4_uld_list, link) {
8283                 if (ui->uld_id == id) {
8284                         rc = ui->deactivate(sc);
8285                         if (rc == 0) {
8286                                 clrbit(&sc->active_ulds, id);
8287                                 ui->refcount--;
8288                         }
8289                         break;
8290                 }
8291         }
8292
8293         sx_sunlock(&t4_uld_list_lock);
8294
8295         return (rc);
8296 }
8297
8298 int
8299 uld_active(struct adapter *sc, int uld_id)
8300 {
8301
8302         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8303
8304         return (isset(&sc->active_ulds, uld_id));
8305 }
8306 #endif
8307
8308 /*
8309  * Come up with reasonable defaults for some of the tunables, provided they're
8310  * not set by the user (in which case we'll use the values as is).
8311  */
8312 static void
8313 tweak_tunables(void)
8314 {
8315         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8316
8317         if (t4_ntxq10g < 1)
8318                 t4_ntxq10g = min(nc, NTXQ_10G);
8319
8320         if (t4_ntxq1g < 1)
8321                 t4_ntxq1g = min(nc, NTXQ_1G);
8322
8323         if (t4_nrxq10g < 1)
8324                 t4_nrxq10g = min(nc, NRXQ_10G);
8325
8326         if (t4_nrxq1g < 1)
8327                 t4_nrxq1g = min(nc, NRXQ_1G);
8328
8329 #ifdef TCP_OFFLOAD
8330         if (t4_nofldtxq10g < 1)
8331                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8332
8333         if (t4_nofldtxq1g < 1)
8334                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8335
8336         if (t4_nofldrxq10g < 1)
8337                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8338
8339         if (t4_nofldrxq1g < 1)
8340                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8341
8342         if (t4_toecaps_allowed == -1)
8343                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8344 #else
8345         if (t4_toecaps_allowed == -1)
8346                 t4_toecaps_allowed = 0;
8347 #endif
8348
8349 #ifdef DEV_NETMAP
8350         if (t4_nnmtxq10g < 1)
8351                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8352
8353         if (t4_nnmtxq1g < 1)
8354                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8355
8356         if (t4_nnmrxq10g < 1)
8357                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8358
8359         if (t4_nnmrxq1g < 1)
8360                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8361 #endif
8362
8363         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8364                 t4_tmr_idx_10g = TMR_IDX_10G;
8365
8366         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8367                 t4_pktc_idx_10g = PKTC_IDX_10G;
8368
8369         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8370                 t4_tmr_idx_1g = TMR_IDX_1G;
8371
8372         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8373                 t4_pktc_idx_1g = PKTC_IDX_1G;
8374
8375         if (t4_qsize_txq < 128)
8376                 t4_qsize_txq = 128;
8377
8378         if (t4_qsize_rxq < 128)
8379                 t4_qsize_rxq = 128;
8380         while (t4_qsize_rxq & 7)
8381                 t4_qsize_rxq++;
8382
8383         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8384 }
8385
8386 static struct sx mlu;   /* mod load unload */
8387 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8388
8389 static int
8390 mod_event(module_t mod, int cmd, void *arg)
8391 {
8392         int rc = 0;
8393         static int loaded = 0;
8394
8395         switch (cmd) {
8396         case MOD_LOAD:
8397                 sx_xlock(&mlu);
8398                 if (loaded++ == 0) {
8399                         t4_sge_modload();
8400                         sx_init(&t4_list_lock, "T4/T5 adapters");
8401                         SLIST_INIT(&t4_list);
8402 #ifdef TCP_OFFLOAD
8403                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8404                         SLIST_INIT(&t4_uld_list);
8405 #endif
8406                         t4_tracer_modload();
8407                         tweak_tunables();
8408                 }
8409                 sx_xunlock(&mlu);
8410                 break;
8411
8412         case MOD_UNLOAD:
8413                 sx_xlock(&mlu);
8414                 if (--loaded == 0) {
8415                         int tries;
8416
8417                         sx_slock(&t4_list_lock);
8418                         if (!SLIST_EMPTY(&t4_list)) {
8419                                 rc = EBUSY;
8420                                 sx_sunlock(&t4_list_lock);
8421                                 goto done_unload;
8422                         }
8423 #ifdef TCP_OFFLOAD
8424                         sx_slock(&t4_uld_list_lock);
8425                         if (!SLIST_EMPTY(&t4_uld_list)) {
8426                                 rc = EBUSY;
8427                                 sx_sunlock(&t4_uld_list_lock);
8428                                 sx_sunlock(&t4_list_lock);
8429                                 goto done_unload;
8430                         }
8431 #endif
8432                         tries = 0;
8433                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8434                                 uprintf("%ju clusters with custom free routine "
8435                                     "still is use.\n", t4_sge_extfree_refs());
8436                                 pause("t4unload", 2 * hz);
8437                         }
8438 #ifdef TCP_OFFLOAD
8439                         sx_sunlock(&t4_uld_list_lock);
8440 #endif
8441                         sx_sunlock(&t4_list_lock);
8442
8443                         if (t4_sge_extfree_refs() == 0) {
8444                                 t4_tracer_modunload();
8445 #ifdef TCP_OFFLOAD
8446                                 sx_destroy(&t4_uld_list_lock);
8447 #endif
8448                                 sx_destroy(&t4_list_lock);
8449                                 t4_sge_modunload();
8450                                 loaded = 0;
8451                         } else {
8452                                 rc = EBUSY;
8453                                 loaded++;       /* undo earlier decrement */
8454                         }
8455                 }
8456 done_unload:
8457                 sx_xunlock(&mlu);
8458                 break;
8459         }
8460
8461         return (rc);
8462 }
8463
8464 static devclass_t t4_devclass, t5_devclass;
8465 static devclass_t cxgbe_devclass, cxl_devclass;
8466
8467 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8468 MODULE_VERSION(t4nex, 1);
8469 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8470
8471 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8472 MODULE_VERSION(t5nex, 1);
8473 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8474
8475 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8476 MODULE_VERSION(cxgbe, 1);
8477
8478 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8479 MODULE_VERSION(cxl, 1);