sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static int
 574 t4_attach(device_t dev)
 575 {
 576         struct adapter *sc;
 577         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 578         struct intrs_and_queues iaq;
 579         struct sge *s;
 580 #ifdef TCP_OFFLOAD
 581         int ofld_rqidx, ofld_tqidx;
 582 #endif
 583 #ifdef DEV_NETMAP
 584         int nm_rqidx, nm_tqidx;
 585 #endif
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 590
 591         pci_enable_busmaster(dev);
 592         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 593                 uint32_t v;
 594
 595                 pci_set_max_read_req(dev, 4096);
 596                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 597                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 598                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 599
 600                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 601         }
 602
 603         sc->traceq = -1;
 604         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 605         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 606             device_get_nameunit(dev));
 607
 608         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 609             device_get_nameunit(dev));
 610         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 611         sx_xlock(&t4_list_lock);
 612         SLIST_INSERT_HEAD(&t4_list, sc, link);
 613         sx_xunlock(&t4_list_lock);
 614
 615         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 616         TAILQ_INIT(&sc->sfl);
 617         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 618
 619         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 620
 621         rc = map_bars_0_and_4(sc);
 622         if (rc != 0)
 623                 goto done; /* error message displayed already */
 624
 625         /*
 626          * This is the real PF# to which we're attaching.  Works from within PCI
 627          * passthrough environments too, where pci_get_function() could return a
 628          * different PF# depending on the passthrough configuration.  We need to
 629          * use the real PF# in all our communication with the firmware.
 630          */
 631         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 632         sc->mbox = sc->pf;
 633
 634         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 635         sc->an_handler = an_not_handled;
 636         for (i = 0; i < nitems(sc->cpl_handler); i++)
 637                 sc->cpl_handler[i] = cpl_not_handled;
 638         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 639                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 640         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 642         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 643         t4_init_sge_cpl_handlers(sc);
 644
 645         /* Prepare the adapter for operation */
 646         rc = -t4_prep_adapter(sc);
 647         if (rc != 0) {
 648                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 649                 goto done;
 650         }
 651
 652         /*
 653          * Do this really early, with the memory windows set up even before the
 654          * character device.  The userland tool's register i/o and mem read
 655          * will work even in "recovery mode".
 656          */
 657         setup_memwin(sc);
 658         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 659             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 660             device_get_nameunit(dev));
 661         if (sc->cdev == NULL)
 662                 device_printf(dev, "failed to create nexus char device.\n");
 663         else
 664                 sc->cdev->si_drv1 = sc;
 665
 666         /* Go no further if recovery mode has been requested. */
 667         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 668                 device_printf(dev, "recovery mode.\n");
 669                 goto done;
 670         }
 671
 672 #if defined(__i386__)
 673         if ((cpu_feature & CPUID_CX8) == 0) {
 674                 device_printf(dev, "64 bit atomics not available.\n");
 675                 rc = ENOTSUP;
 676                 goto done;
 677         }
 678 #endif
 679
 680         /* Prepare the firmware for operation */
 681         rc = prep_firmware(sc);
 682         if (rc != 0)
 683                 goto done; /* error message displayed already */
 684
 685         rc = get_params__post_init(sc);
 686         if (rc != 0)
 687                 goto done; /* error message displayed already */
 688
 689         rc = set_params__post_init(sc);
 690         if (rc != 0)
 691                 goto done; /* error message displayed already */
 692
 693         rc = map_bar_2(sc);
 694         if (rc != 0)
 695                 goto done; /* error message displayed already */
 696
 697         rc = t4_create_dma_tag(sc);
 698         if (rc != 0)
 699                 goto done; /* error message displayed already */
 700
 701         /*
 702          * First pass over all the ports - allocate VIs and initialize some
 703          * basic parameters like mac address, port type, etc.  We also figure
 704          * out whether a port is 10G or 1G and use that information when
 705          * calculating how many interrupts to attempt to allocate.
 706          */
 707         n10g = n1g = 0;
 708         for_each_port(sc, i) {
 709                 struct port_info *pi;
 710
 711                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 712                 sc->port[i] = pi;
 713
 714                 /* These must be set before t4_port_init */
 715                 pi->adapter = sc;
 716                 pi->port_id = i;
 717
 718                 /* Allocate the vi and initialize parameters like mac addr */
 719                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 720                 if (rc != 0) {
 721                         device_printf(dev, "unable to initialize port %d: %d\n",
 722                             i, rc);
 723                         free(pi, M_CXGBE);
 724                         sc->port[i] = NULL;
 725                         goto done;
 726                 }
 727
 728                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 729                 pi->link_cfg.requested_fc |= t4_pause_settings;
 730                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 731                 pi->link_cfg.fc |= t4_pause_settings;
 732
 733                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 734                 if (rc != 0) {
 735                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 736                         free(pi, M_CXGBE);
 737                         sc->port[i] = NULL;
 738                         goto done;
 739                 }
 740
 741                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 742                     device_get_nameunit(dev), i);
 743                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 744                 sc->chan_map[pi->tx_chan] = i;
 745
 746                 if (is_10G_port(pi) || is_40G_port(pi)) {
 747                         n10g++;
 748                         pi->tmr_idx = t4_tmr_idx_10g;
 749                         pi->pktc_idx = t4_pktc_idx_10g;
 750                 } else {
 751                         n1g++;
 752                         pi->tmr_idx = t4_tmr_idx_1g;
 753                         pi->pktc_idx = t4_pktc_idx_1g;
 754                 }
 755
 756                 pi->xact_addr_filt = -1;
 757                 pi->linkdnrc = -1;
 758
 759                 pi->qsize_rxq = t4_qsize_rxq;
 760                 pi->qsize_txq = t4_qsize_txq;
 761
 762                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 763                 if (pi->dev == NULL) {
 764                         device_printf(dev,
 765                             "failed to add device for port %d.\n", i);
 766                         rc = ENXIO;
 767                         goto done;
 768                 }
 769                 device_set_softc(pi->dev, pi);
 770         }
 771
 772         /*
 773          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 774          */
 775         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 776         if (rc != 0)
 777                 goto done; /* error message displayed already */
 778
 779         sc->intr_type = iaq.intr_type;
 780         sc->intr_count = iaq.nirq;
 781
 782         s = &sc->sge;
 783         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 784         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 785         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 786         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 787         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 788 #ifdef TCP_OFFLOAD
 789         if (is_offload(sc)) {
 790                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 791                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 792                 s->neq += s->nofldtxq + s->nofldrxq;
 793                 s->niq += s->nofldrxq;
 794
 795                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 796                     M_CXGBE, M_ZERO | M_WAITOK);
 797                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 798                     M_CXGBE, M_ZERO | M_WAITOK);
 799         }
 800 #endif
 801 #ifdef DEV_NETMAP
 802         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 803         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 804         s->neq += s->nnmtxq + s->nnmrxq;
 805         s->niq += s->nnmrxq;
 806
 807         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 808             M_CXGBE, M_ZERO | M_WAITOK);
 809         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 810             M_CXGBE, M_ZERO | M_WAITOK);
 811 #endif
 812
 813         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 814             M_ZERO | M_WAITOK);
 815         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 816             M_ZERO | M_WAITOK);
 817         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 818             M_ZERO | M_WAITOK);
 819         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 820             M_ZERO | M_WAITOK);
 821         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 822             M_ZERO | M_WAITOK);
 823
 824         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 825             M_ZERO | M_WAITOK);
 826
 827         t4_init_l2t(sc, M_WAITOK);
 828
 829         /*
 830          * Second pass over the ports.  This time we know the number of rx and
 831          * tx queues that each port should get.
 832          */
 833         rqidx = tqidx = 0;
 834 #ifdef TCP_OFFLOAD
 835         ofld_rqidx = ofld_tqidx = 0;
 836 #endif
 837 #ifdef DEV_NETMAP
 838         nm_rqidx = nm_tqidx = 0;
 839 #endif
 840         for_each_port(sc, i) {
 841                 struct port_info *pi = sc->port[i];
 842
 843                 if (pi == NULL)
 844                         continue;
 845
 846                 pi->first_rxq = rqidx;
 847                 pi->first_txq = tqidx;
 848                 if (is_10G_port(pi) || is_40G_port(pi)) {
 849                         pi->flags |= iaq.intr_flags_10g;
 850                         pi->nrxq = iaq.nrxq10g;
 851                         pi->ntxq = iaq.ntxq10g;
 852                 } else {
 853                         pi->flags |= iaq.intr_flags_1g;
 854                         pi->nrxq = iaq.nrxq1g;
 855                         pi->ntxq = iaq.ntxq1g;
 856                 }
 857
 858                 if (pi->ntxq > 1)
 859                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 860                 else
 861                         pi->rsrv_noflowq = 0;
 862
 863                 rqidx += pi->nrxq;
 864                 tqidx += pi->ntxq;
 865 #ifdef TCP_OFFLOAD
 866                 if (is_offload(sc)) {
 867                         pi->first_ofld_rxq = ofld_rqidx;
 868                         pi->first_ofld_txq = ofld_tqidx;
 869                         if (is_10G_port(pi) || is_40G_port(pi)) {
 870                                 pi->nofldrxq = iaq.nofldrxq10g;
 871                                 pi->nofldtxq = iaq.nofldtxq10g;
 872                         } else {
 873                                 pi->nofldrxq = iaq.nofldrxq1g;
 874                                 pi->nofldtxq = iaq.nofldtxq1g;
 875                         }
 876                         ofld_rqidx += pi->nofldrxq;
 877                         ofld_tqidx += pi->nofldtxq;
 878                 }
 879 #endif
 880 #ifdef DEV_NETMAP
 881                 pi->first_nm_rxq = nm_rqidx;
 882                 pi->first_nm_txq = nm_tqidx;
 883                 if (is_10G_port(pi) || is_40G_port(pi)) {
 884                         pi->nnmrxq = iaq.nnmrxq10g;
 885                         pi->nnmtxq = iaq.nnmtxq10g;
 886                 } else {
 887                         pi->nnmrxq = iaq.nnmrxq1g;
 888                         pi->nnmtxq = iaq.nnmtxq1g;
 889                 }
 890                 nm_rqidx += pi->nnmrxq;
 891                 nm_tqidx += pi->nnmtxq;
 892 #endif
 893         }
 894
 895         rc = setup_intr_handlers(sc);
 896         if (rc != 0) {
 897                 device_printf(dev,
 898                     "failed to setup interrupt handlers: %d\n", rc);
 899                 goto done;
 900         }
 901
 902         rc = bus_generic_attach(dev);
 903         if (rc != 0) {
 904                 device_printf(dev,
 905                     "failed to attach all child ports: %d\n", rc);
 906                 goto done;
 907         }
 908
 909         device_printf(dev,
 910             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 911             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 912             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 913             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 914             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 915
 916         t4_set_desc(sc);
 917
 918 done:
 919         if (rc != 0 && sc->cdev) {
 920                 /* cdev was created and so cxgbetool works; recover that way. */
 921                 device_printf(dev,
 922                     "error during attach, adapter is now in recovery mode.\n");
 923                 rc = 0;
 924         }
 925
 926         if (rc != 0)
 927                 t4_detach(dev);
 928         else
 929                 t4_sysctls(sc);
 930
 931         return (rc);
 932 }
 933
 934 /*
 935  * Idempotent
 936  */
 937 static int
 938 t4_detach(device_t dev)
 939 {
 940         struct adapter *sc;
 941         struct port_info *pi;
 942         int i, rc;
 943
 944         sc = device_get_softc(dev);
 945
 946         if (sc->flags & FULL_INIT_DONE)
 947                 t4_intr_disable(sc);
 948
 949         if (sc->cdev) {
 950                 destroy_dev(sc->cdev);
 951                 sc->cdev = NULL;
 952         }
 953
 954         rc = bus_generic_detach(dev);
 955         if (rc) {
 956                 device_printf(dev,
 957                     "failed to detach child devices: %d\n", rc);
 958                 return (rc);
 959         }
 960
 961         for (i = 0; i < sc->intr_count; i++)
 962                 t4_free_irq(sc, &sc->irq[i]);
 963
 964         for (i = 0; i < MAX_NPORTS; i++) {
 965                 pi = sc->port[i];
 966                 if (pi) {
 967                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 968                         if (pi->dev)
 969                                 device_delete_child(dev, pi->dev);
 970
 971                         mtx_destroy(&pi->pi_lock);
 972                         free(pi, M_CXGBE);
 973                 }
 974         }
 975
 976         if (sc->flags & FULL_INIT_DONE)
 977                 adapter_full_uninit(sc);
 978
 979         if (sc->flags & FW_OK)
 980                 t4_fw_bye(sc, sc->mbox);
 981
 982         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 983                 pci_release_msi(dev);
 984
 985         if (sc->regs_res)
 986                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 987                     sc->regs_res);
 988
 989         if (sc->udbs_res)
 990                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 991                     sc->udbs_res);
 992
 993         if (sc->msix_res)
 994                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 995                     sc->msix_res);
 996
 997         if (sc->l2t)
 998                 t4_free_l2t(sc->l2t);
 999
1000 #ifdef TCP_OFFLOAD
1001         free(sc->sge.ofld_rxq, M_CXGBE);
1002         free(sc->sge.ofld_txq, M_CXGBE);
1003 #endif
1004 #ifdef DEV_NETMAP
1005         free(sc->sge.nm_rxq, M_CXGBE);
1006         free(sc->sge.nm_txq, M_CXGBE);
1007 #endif
1008         free(sc->irq, M_CXGBE);
1009         free(sc->sge.rxq, M_CXGBE);
1010         free(sc->sge.txq, M_CXGBE);
1011         free(sc->sge.ctrlq, M_CXGBE);
1012         free(sc->sge.iqmap, M_CXGBE);
1013         free(sc->sge.eqmap, M_CXGBE);
1014         free(sc->tids.ftid_tab, M_CXGBE);
1015         t4_destroy_dma_tag(sc);
1016         if (mtx_initialized(&sc->sc_lock)) {
1017                 sx_xlock(&t4_list_lock);
1018                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1019                 sx_xunlock(&t4_list_lock);
1020                 mtx_destroy(&sc->sc_lock);
1021         }
1022
1023         if (mtx_initialized(&sc->tids.ftid_lock))
1024                 mtx_destroy(&sc->tids.ftid_lock);
1025         if (mtx_initialized(&sc->sfl_lock))
1026                 mtx_destroy(&sc->sfl_lock);
1027         if (mtx_initialized(&sc->ifp_lock))
1028                 mtx_destroy(&sc->ifp_lock);
1029         if (mtx_initialized(&sc->regwin_lock))
1030                 mtx_destroy(&sc->regwin_lock);
1031
1032         bzero(sc, sizeof(*sc));
1033
1034         return (0);
1035 }
1036
1037 static int
1038 cxgbe_probe(device_t dev)
1039 {
1040         char buf[128];
1041         struct port_info *pi = device_get_softc(dev);
1042
1043         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1044         device_set_desc_copy(dev, buf);
1045
1046         return (BUS_PROBE_DEFAULT);
1047 }
1048
1049 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1050     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1051     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1052 #define T4_CAP_ENABLE (T4_CAP)
1053
1054 static int
1055 cxgbe_attach(device_t dev)
1056 {
1057         struct port_info *pi = device_get_softc(dev);
1058         struct ifnet *ifp;
1059         char *s;
1060         int n, o;
1061
1062         /* Allocate an ifnet and set it up */
1063         ifp = if_alloc(IFT_ETHER);
1064         if (ifp == NULL) {
1065                 device_printf(dev, "Cannot allocate ifnet\n");
1066                 return (ENOMEM);
1067         }
1068         pi->ifp = ifp;
1069         ifp->if_softc = pi;
1070
1071         callout_init(&pi->tick, CALLOUT_MPSAFE);
1072
1073         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1074         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1075
1076         ifp->if_init = cxgbe_init;
1077         ifp->if_ioctl = cxgbe_ioctl;
1078         ifp->if_transmit = cxgbe_transmit;
1079         ifp->if_qflush = cxgbe_qflush;
1080
1081         ifp->if_capabilities = T4_CAP;
1082 #ifdef TCP_OFFLOAD
1083         if (is_offload(pi->adapter))
1084                 ifp->if_capabilities |= IFCAP_TOE;
1085 #endif
1086         ifp->if_capenable = T4_CAP_ENABLE;
1087         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1088             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1089
1090         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1091         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1092         ifp->if_hw_tsomaxsegsize = 65536;
1093
1094         /* Initialize ifmedia for this port */
1095         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1096             cxgbe_media_status);
1097         build_medialist(pi, &pi->media);
1098
1099         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1100             EVENTHANDLER_PRI_ANY);
1101
1102         ether_ifattach(ifp, pi->hw_addr);
1103
1104         n = 128;
1105         s = malloc(n, M_CXGBE, M_WAITOK);
1106         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1107         MPASS(n > o);
1108 #ifdef TCP_OFFLOAD
1109         if (is_offload(pi->adapter)) {
1110                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1111                     pi->nofldtxq, pi->nofldrxq);
1112                 MPASS(n > o);
1113         }
1114 #endif
1115 #ifdef DEV_NETMAP
1116         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1117             pi->nnmrxq);
1118         MPASS(n > o);
1119 #endif
1120         device_printf(dev, "%s\n", s);
1121         free(s, M_CXGBE);
1122
1123 #ifdef DEV_NETMAP
1124         /* nm_media handled here to keep implementation private to this file */
1125         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1126             cxgbe_media_status);
1127         build_medialist(pi, &pi->nm_media);
1128         create_netmap_ifnet(pi);        /* logs errors it something fails */
1129 #endif
1130         cxgbe_sysctls(pi);
1131
1132         return (0);
1133 }
1134
1135 static int
1136 cxgbe_detach(device_t dev)
1137 {
1138         struct port_info *pi = device_get_softc(dev);
1139         struct adapter *sc = pi->adapter;
1140         struct ifnet *ifp = pi->ifp;
1141
1142         /* Tell if_ioctl and if_init that the port is going away */
1143         ADAPTER_LOCK(sc);
1144         SET_DOOMED(pi);
1145         wakeup(&sc->flags);
1146         while (IS_BUSY(sc))
1147                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1148         SET_BUSY(sc);
1149 #ifdef INVARIANTS
1150         sc->last_op = "t4detach";
1151         sc->last_op_thr = curthread;
1152 #endif
1153         ADAPTER_UNLOCK(sc);
1154
1155         if (pi->flags & HAS_TRACEQ) {
1156                 sc->traceq = -1;        /* cloner should not create ifnet */
1157                 t4_tracer_port_detach(sc);
1158         }
1159
1160         if (pi->vlan_c)
1161                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1162
1163         PORT_LOCK(pi);
1164         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1165         callout_stop(&pi->tick);
1166         PORT_UNLOCK(pi);
1167         callout_drain(&pi->tick);
1168
1169         /* Let detach proceed even if these fail. */
1170         cxgbe_uninit_synchronized(pi);
1171         port_full_uninit(pi);
1172
1173         ifmedia_removeall(&pi->media);
1174         ether_ifdetach(pi->ifp);
1175         if_free(pi->ifp);
1176
1177 #ifdef DEV_NETMAP
1178         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1179         destroy_netmap_ifnet(pi);
1180 #endif
1181
1182         ADAPTER_LOCK(sc);
1183         CLR_BUSY(sc);
1184         wakeup(&sc->flags);
1185         ADAPTER_UNLOCK(sc);
1186
1187         return (0);
1188 }
1189
1190 static void
1191 cxgbe_init(void *arg)
1192 {
1193         struct port_info *pi = arg;
1194         struct adapter *sc = pi->adapter;
1195
1196         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1197                 return;
1198         cxgbe_init_synchronized(pi);
1199         end_synchronized_op(sc, 0);
1200 }
1201
1202 static int
1203 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1204 {
1205         int rc = 0, mtu, flags, can_sleep;
1206         struct port_info *pi = ifp->if_softc;
1207         struct adapter *sc = pi->adapter;
1208         struct ifreq *ifr = (struct ifreq *)data;
1209         uint32_t mask;
1210
1211         switch (cmd) {
1212         case SIOCSIFMTU:
1213                 mtu = ifr->ifr_mtu;
1214                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1215                         return (EINVAL);
1216
1217                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1218                 if (rc)
1219                         return (rc);
1220                 ifp->if_mtu = mtu;
1221                 if (pi->flags & PORT_INIT_DONE) {
1222                         t4_update_fl_bufsize(ifp);
1223                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1224                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1225                 }
1226                 end_synchronized_op(sc, 0);
1227                 break;
1228
1229         case SIOCSIFFLAGS:
1230                 can_sleep = 0;
1231 redo_sifflags:
1232                 rc = begin_synchronized_op(sc, pi,
1233                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1234                 if (rc)
1235                         return (rc);
1236
1237                 if (ifp->if_flags & IFF_UP) {
1238                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1239                                 flags = pi->if_flags;
1240                                 if ((ifp->if_flags ^ flags) &
1241                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1242                                         if (can_sleep == 1) {
1243                                                 end_synchronized_op(sc, 0);
1244                                                 can_sleep = 0;
1245                                                 goto redo_sifflags;
1246                                         }
1247                                         rc = update_mac_settings(ifp,
1248                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1249                                 }
1250                         } else {
1251                                 if (can_sleep == 0) {
1252                                         end_synchronized_op(sc, LOCK_HELD);
1253                                         can_sleep = 1;
1254                                         goto redo_sifflags;
1255                                 }
1256                                 rc = cxgbe_init_synchronized(pi);
1257                         }
1258                         pi->if_flags = ifp->if_flags;
1259                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1260                         if (can_sleep == 0) {
1261                                 end_synchronized_op(sc, LOCK_HELD);
1262                                 can_sleep = 1;
1263                                 goto redo_sifflags;
1264                         }
1265                         rc = cxgbe_uninit_synchronized(pi);
1266                 }
1267                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1268                 break;
1269
1270         case SIOCADDMULTI:
1271         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1272                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1273                 if (rc)
1274                         return (rc);
1275                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1276                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1277                 end_synchronized_op(sc, LOCK_HELD);
1278                 break;
1279
1280         case SIOCSIFCAP:
1281                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1282                 if (rc)
1283                         return (rc);
1284
1285                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1286                 if (mask & IFCAP_TXCSUM) {
1287                         ifp->if_capenable ^= IFCAP_TXCSUM;
1288                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1289
1290                         if (IFCAP_TSO4 & ifp->if_capenable &&
1291                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1292                                 ifp->if_capenable &= ~IFCAP_TSO4;
1293                                 if_printf(ifp,
1294                                     "tso4 disabled due to -txcsum.\n");
1295                         }
1296                 }
1297                 if (mask & IFCAP_TXCSUM_IPV6) {
1298                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1299                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1300
1301                         if (IFCAP_TSO6 & ifp->if_capenable &&
1302                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1303                                 ifp->if_capenable &= ~IFCAP_TSO6;
1304                                 if_printf(ifp,
1305                                     "tso6 disabled due to -txcsum6.\n");
1306                         }
1307                 }
1308                 if (mask & IFCAP_RXCSUM)
1309                         ifp->if_capenable ^= IFCAP_RXCSUM;
1310                 if (mask & IFCAP_RXCSUM_IPV6)
1311                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1312
1313                 /*
1314                  * Note that we leave CSUM_TSO alone (it is always set).  The
1315                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1316                  * sending a TSO request our way, so it's sufficient to toggle
1317                  * IFCAP_TSOx only.
1318                  */
1319                 if (mask & IFCAP_TSO4) {
1320                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1321                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1322                                 if_printf(ifp, "enable txcsum first.\n");
1323                                 rc = EAGAIN;
1324                                 goto fail;
1325                         }
1326                         ifp->if_capenable ^= IFCAP_TSO4;
1327                 }
1328                 if (mask & IFCAP_TSO6) {
1329                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1330                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1331                                 if_printf(ifp, "enable txcsum6 first.\n");
1332                                 rc = EAGAIN;
1333                                 goto fail;
1334                         }
1335                         ifp->if_capenable ^= IFCAP_TSO6;
1336                 }
1337                 if (mask & IFCAP_LRO) {
1338 #if defined(INET) || defined(INET6)
1339                         int i;
1340                         struct sge_rxq *rxq;
1341
1342                         ifp->if_capenable ^= IFCAP_LRO;
1343                         for_each_rxq(pi, i, rxq) {
1344                                 if (ifp->if_capenable & IFCAP_LRO)
1345                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1346                                 else
1347                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1348                         }
1349 #endif
1350                 }
1351 #ifdef TCP_OFFLOAD
1352                 if (mask & IFCAP_TOE) {
1353                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1354
1355                         rc = toe_capability(pi, enable);
1356                         if (rc != 0)
1357                                 goto fail;
1358
1359                         ifp->if_capenable ^= mask;
1360                 }
1361 #endif
1362                 if (mask & IFCAP_VLAN_HWTAGGING) {
1363                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1364                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1365                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1366                 }
1367                 if (mask & IFCAP_VLAN_MTU) {
1368                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1369
1370                         /* Need to find out how to disable auto-mtu-inflation */
1371                 }
1372                 if (mask & IFCAP_VLAN_HWTSO)
1373                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1374                 if (mask & IFCAP_VLAN_HWCSUM)
1375                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1376
1377 #ifdef VLAN_CAPABILITIES
1378                 VLAN_CAPABILITIES(ifp);
1379 #endif
1380 fail:
1381                 end_synchronized_op(sc, 0);
1382                 break;
1383
1384         case SIOCSIFMEDIA:
1385         case SIOCGIFMEDIA:
1386                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1387                 break;
1388
1389         case SIOCGI2C: {
1390                 struct ifi2creq i2c;
1391
1392                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1393                 if (rc != 0)
1394                         break;
1395                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1396                         rc = EPERM;
1397                         break;
1398                 }
1399                 if (i2c.len > sizeof(i2c.data)) {
1400                         rc = EINVAL;
1401                         break;
1402                 }
1403                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1404                 if (rc)
1405                         return (rc);
1406                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1407                     i2c.offset, i2c.len, &i2c.data[0]);
1408                 end_synchronized_op(sc, 0);
1409                 if (rc == 0)
1410                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1411                 break;
1412         }
1413
1414         default:
1415                 rc = ether_ioctl(ifp, cmd, data);
1416         }
1417
1418         return (rc);
1419 }
1420
1421 static int
1422 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1423 {
1424         struct port_info *pi = ifp->if_softc;
1425         struct adapter *sc = pi->adapter;
1426         struct sge_txq *txq;
1427         void *items[1];
1428         int rc;
1429
1430         M_ASSERTPKTHDR(m);
1431         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1432
1433         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1434                 m_freem(m);
1435                 return (ENETDOWN);
1436         }
1437
1438         rc = parse_pkt(&m);
1439         if (__predict_false(rc != 0)) {
1440                 MPASS(m == NULL);                       /* was freed already */
1441                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1442                 return (rc);
1443         }
1444
1445         /* Select a txq. */
1446         txq = &sc->sge.txq[pi->first_txq];
1447         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1448                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1449                     pi->rsrv_noflowq);
1450
1451         items[0] = m;
1452         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1453         if (__predict_false(rc != 0))
1454                 m_freem(m);
1455
1456         return (rc);
1457 }
1458
1459 static void
1460 cxgbe_qflush(struct ifnet *ifp)
1461 {
1462         struct port_info *pi = ifp->if_softc;
1463         struct sge_txq *txq;
1464         int i;
1465
1466         /* queues do not exist if !PORT_INIT_DONE. */
1467         if (pi->flags & PORT_INIT_DONE) {
1468                 for_each_txq(pi, i, txq) {
1469                         TXQ_LOCK(txq);
1470                         txq->eq.flags &= ~EQ_ENABLED;
1471                         TXQ_UNLOCK(txq);
1472                         while (!mp_ring_is_idle(txq->r)) {
1473                                 mp_ring_check_drainage(txq->r, 0);
1474                                 pause("qflush", 1);
1475                         }
1476                 }
1477         }
1478         if_qflush(ifp);
1479 }
1480
1481 static int
1482 cxgbe_media_change(struct ifnet *ifp)
1483 {
1484         struct port_info *pi = ifp->if_softc;
1485
1486         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1487
1488         return (EOPNOTSUPP);
1489 }
1490
1491 static void
1492 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1493 {
1494         struct port_info *pi = ifp->if_softc;
1495         struct ifmedia *media = NULL;
1496         struct ifmedia_entry *cur;
1497         int speed = pi->link_cfg.speed;
1498 #ifdef INVARIANTS
1499         int data = (pi->port_type << 8) | pi->mod_type;
1500 #endif
1501
1502         if (ifp == pi->ifp)
1503                 media = &pi->media;
1504 #ifdef DEV_NETMAP
1505         else if (ifp == pi->nm_ifp)
1506                 media = &pi->nm_media;
1507 #endif
1508         MPASS(media != NULL);
1509
1510         cur = media->ifm_cur;
1511         MPASS(cur->ifm_data == data);
1512
1513         ifmr->ifm_status = IFM_AVALID;
1514         if (!pi->link_cfg.link_ok)
1515                 return;
1516
1517         ifmr->ifm_status |= IFM_ACTIVE;
1518
1519         /* active and current will differ iff current media is autoselect. */
1520         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1521                 return;
1522
1523         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1524         if (speed == SPEED_10000)
1525                 ifmr->ifm_active |= IFM_10G_T;
1526         else if (speed == SPEED_1000)
1527                 ifmr->ifm_active |= IFM_1000_T;
1528         else if (speed == SPEED_100)
1529                 ifmr->ifm_active |= IFM_100_TX;
1530         else if (speed == SPEED_10)
1531                 ifmr->ifm_active |= IFM_10_T;
1532         else
1533                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1534                             speed));
1535 }
1536
1537 void
1538 t4_fatal_err(struct adapter *sc)
1539 {
1540         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1541         t4_intr_disable(sc);
1542         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1543             device_get_nameunit(sc->dev));
1544 }
1545
1546 static int
1547 map_bars_0_and_4(struct adapter *sc)
1548 {
1549         sc->regs_rid = PCIR_BAR(0);
1550         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1551             &sc->regs_rid, RF_ACTIVE);
1552         if (sc->regs_res == NULL) {
1553                 device_printf(sc->dev, "cannot map registers.\n");
1554                 return (ENXIO);
1555         }
1556         sc->bt = rman_get_bustag(sc->regs_res);
1557         sc->bh = rman_get_bushandle(sc->regs_res);
1558         sc->mmio_len = rman_get_size(sc->regs_res);
1559         setbit(&sc->doorbells, DOORBELL_KDB);
1560
1561         sc->msix_rid = PCIR_BAR(4);
1562         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1563             &sc->msix_rid, RF_ACTIVE);
1564         if (sc->msix_res == NULL) {
1565                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1566                 return (ENXIO);
1567         }
1568
1569         return (0);
1570 }
1571
1572 static int
1573 map_bar_2(struct adapter *sc)
1574 {
1575
1576         /*
1577          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1578          * to map it if RDMA is disabled.
1579          */
1580         if (is_t4(sc) && sc->rdmacaps == 0)
1581                 return (0);
1582
1583         sc->udbs_rid = PCIR_BAR(2);
1584         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1585             &sc->udbs_rid, RF_ACTIVE);
1586         if (sc->udbs_res == NULL) {
1587                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1588                 return (ENXIO);
1589         }
1590         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1591
1592         if (is_t5(sc)) {
1593                 setbit(&sc->doorbells, DOORBELL_UDB);
1594 #if defined(__i386__) || defined(__amd64__)
1595                 if (t5_write_combine) {
1596                         int rc;
1597
1598                         /*
1599                          * Enable write combining on BAR2.  This is the
1600                          * userspace doorbell BAR and is split into 128B
1601                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1602                          * with an egress queue.  The first 64B has the doorbell
1603                          * and the second 64B can be used to submit a tx work
1604                          * request with an implicit doorbell.
1605                          */
1606
1607                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1608                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1609                         if (rc == 0) {
1610                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1611                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1612                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1613                         } else {
1614                                 device_printf(sc->dev,
1615                                     "couldn't enable write combining: %d\n",
1616                                     rc);
1617                         }
1618
1619                         t4_write_reg(sc, A_SGE_STAT_CFG,
1620                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1621                 }
1622 #endif
1623         }
1624
1625         return (0);
1626 }
1627
1628 static const struct memwin t4_memwin[] = {
1629         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1630         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1631         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1632 };
1633
1634 static const struct memwin t5_memwin[] = {
1635         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1636         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1637         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1638 };
1639
1640 static void
1641 setup_memwin(struct adapter *sc)
1642 {
1643         const struct memwin *mw;
1644         int i, n;
1645         uint32_t bar0;
1646
1647         if (is_t4(sc)) {
1648                 /*
1649                  * Read low 32b of bar0 indirectly via the hardware backdoor
1650                  * mechanism.  Works from within PCI passthrough environments
1651                  * too, where rman_get_start() can return a different value.  We
1652                  * need to program the T4 memory window decoders with the actual
1653                  * addresses that will be coming across the PCIe link.
1654                  */
1655                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1656                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1657
1658                 mw = &t4_memwin[0];
1659                 n = nitems(t4_memwin);
1660         } else {
1661                 /* T5 uses the relative offset inside the PCIe BAR */
1662                 bar0 = 0;
1663
1664                 mw = &t5_memwin[0];
1665                 n = nitems(t5_memwin);
1666         }
1667
1668         for (i = 0; i < n; i++, mw++) {
1669                 t4_write_reg(sc,
1670                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1671                     (mw->base + bar0) | V_BIR(0) |
1672                     V_WINDOW(ilog2(mw->aperture) - 10));
1673         }
1674
1675         /* flush */
1676         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1677 }
1678
1679 /*
1680  * Verify that the memory range specified by the addr/len pair is valid and lies
1681  * entirely within a single region (EDCx or MCx).
1682  */
1683 static int
1684 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1685 {
1686         uint32_t em, addr_len, maddr, mlen;
1687
1688         /* Memory can only be accessed in naturally aligned 4 byte units */
1689         if (addr & 3 || len & 3 || len == 0)
1690                 return (EINVAL);
1691
1692         /* Enabled memories */
1693         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1694         if (em & F_EDRAM0_ENABLE) {
1695                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1696                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1697                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1698                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1699                     addr + len <= maddr + mlen)
1700                         return (0);
1701         }
1702         if (em & F_EDRAM1_ENABLE) {
1703                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1704                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1705                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1706                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1707                     addr + len <= maddr + mlen)
1708                         return (0);
1709         }
1710         if (em & F_EXT_MEM_ENABLE) {
1711                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1712                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1713                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1714                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1715                     addr + len <= maddr + mlen)
1716                         return (0);
1717         }
1718         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1719                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1720                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1721                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1722                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1723                     addr + len <= maddr + mlen)
1724                         return (0);
1725         }
1726
1727         return (EFAULT);
1728 }
1729
1730 static int
1731 fwmtype_to_hwmtype(int mtype)
1732 {
1733
1734         switch (mtype) {
1735         case FW_MEMTYPE_EDC0:
1736                 return (MEM_EDC0);
1737         case FW_MEMTYPE_EDC1:
1738                 return (MEM_EDC1);
1739         case FW_MEMTYPE_EXTMEM:
1740                 return (MEM_MC0);
1741         case FW_MEMTYPE_EXTMEM1:
1742                 return (MEM_MC1);
1743         default:
1744                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1745         }
1746 }
1747
1748 /*
1749  * Verify that the memory range specified by the memtype/offset/len pair is
1750  * valid and lies entirely within the memtype specified.  The global address of
1751  * the start of the range is returned in addr.
1752  */
1753 static int
1754 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1755     uint32_t *addr)
1756 {
1757         uint32_t em, addr_len, maddr, mlen;
1758
1759         /* Memory can only be accessed in naturally aligned 4 byte units */
1760         if (off & 3 || len & 3 || len == 0)
1761                 return (EINVAL);
1762
1763         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1764         switch (fwmtype_to_hwmtype(mtype)) {
1765         case MEM_EDC0:
1766                 if (!(em & F_EDRAM0_ENABLE))
1767                         return (EINVAL);
1768                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1769                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1770                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1771                 break;
1772         case MEM_EDC1:
1773                 if (!(em & F_EDRAM1_ENABLE))
1774                         return (EINVAL);
1775                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1776                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1777                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1778                 break;
1779         case MEM_MC:
1780                 if (!(em & F_EXT_MEM_ENABLE))
1781                         return (EINVAL);
1782                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1783                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1784                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1785                 break;
1786         case MEM_MC1:
1787                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1788                         return (EINVAL);
1789                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1790                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1791                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1792                 break;
1793         default:
1794                 return (EINVAL);
1795         }
1796
1797         if (mlen > 0 && off < mlen && off + len <= mlen) {
1798                 *addr = maddr + off;    /* global address */
1799                 return (0);
1800         }
1801
1802         return (EFAULT);
1803 }
1804
1805 static void
1806 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1807 {
1808         const struct memwin *mw;
1809
1810         if (is_t4(sc)) {
1811                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1812                     ("%s: incorrect memwin# (%d)", __func__, win));
1813                 mw = &t4_memwin[win];
1814         } else {
1815                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1816                     ("%s: incorrect memwin# (%d)", __func__, win));
1817                 mw = &t5_memwin[win];
1818         }
1819
1820         if (base != NULL)
1821                 *base = mw->base;
1822         if (aperture != NULL)
1823                 *aperture = mw->aperture;
1824 }
1825
1826 /*
1827  * Positions the memory window such that it can be used to access the specified
1828  * address in the chip's address space.  The return value is the offset of addr
1829  * from the start of the window.
1830  */
1831 static uint32_t
1832 position_memwin(struct adapter *sc, int n, uint32_t addr)
1833 {
1834         uint32_t start, pf;
1835         uint32_t reg;
1836
1837         KASSERT(n >= 0 && n <= 3,
1838             ("%s: invalid window %d.", __func__, n));
1839         KASSERT((addr & 3) == 0,
1840             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1841
1842         if (is_t4(sc)) {
1843                 pf = 0;
1844                 start = addr & ~0xf;    /* start must be 16B aligned */
1845         } else {
1846                 pf = V_PFNUM(sc->pf);
1847                 start = addr & ~0x7f;   /* start must be 128B aligned */
1848         }
1849         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1850
1851         t4_write_reg(sc, reg, start | pf);
1852         t4_read_reg(sc, reg);
1853
1854         return (addr - start);
1855 }
1856
1857 static int
1858 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1859     struct intrs_and_queues *iaq)
1860 {
1861         int rc, itype, navail, nrxq10g, nrxq1g, n;
1862         int nofldrxq10g = 0, nofldrxq1g = 0;
1863         int nnmrxq10g = 0, nnmrxq1g = 0;
1864
1865         bzero(iaq, sizeof(*iaq));
1866
1867         iaq->ntxq10g = t4_ntxq10g;
1868         iaq->ntxq1g = t4_ntxq1g;
1869         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1870         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1871         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1872 #ifdef TCP_OFFLOAD
1873         if (is_offload(sc)) {
1874                 iaq->nofldtxq10g = t4_nofldtxq10g;
1875                 iaq->nofldtxq1g = t4_nofldtxq1g;
1876                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1877                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1878         }
1879 #endif
1880 #ifdef DEV_NETMAP
1881         iaq->nnmtxq10g = t4_nnmtxq10g;
1882         iaq->nnmtxq1g = t4_nnmtxq1g;
1883         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1884         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1885 #endif
1886
1887         for (itype = INTR_MSIX; itype; itype >>= 1) {
1888
1889                 if ((itype & t4_intr_types) == 0)
1890                         continue;       /* not allowed */
1891
1892                 if (itype == INTR_MSIX)
1893                         navail = pci_msix_count(sc->dev);
1894                 else if (itype == INTR_MSI)
1895                         navail = pci_msi_count(sc->dev);
1896                 else
1897                         navail = 1;
1898 restart:
1899                 if (navail == 0)
1900                         continue;
1901
1902                 iaq->intr_type = itype;
1903                 iaq->intr_flags_10g = 0;
1904                 iaq->intr_flags_1g = 0;
1905
1906                 /*
1907                  * Best option: an interrupt vector for errors, one for the
1908                  * firmware event queue, and one for every rxq (NIC, TOE, and
1909                  * netmap).
1910                  */
1911                 iaq->nirq = T4_EXTRA_INTR;
1912                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1913                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1914                 if (iaq->nirq <= navail &&
1915                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1916                         iaq->intr_flags_10g = INTR_ALL;
1917                         iaq->intr_flags_1g = INTR_ALL;
1918                         goto allocate;
1919                 }
1920
1921                 /*
1922                  * Second best option: a vector for errors, one for the firmware
1923                  * event queue, and vectors for either all the NIC rx queues or
1924                  * all the TOE rx queues.  The queues that don't get vectors
1925                  * will forward their interrupts to those that do.
1926                  *
1927                  * Note: netmap rx queues cannot be created early and so they
1928                  * can't be setup to receive forwarded interrupts for others.
1929                  */
1930                 iaq->nirq = T4_EXTRA_INTR;
1931                 if (nrxq10g >= nofldrxq10g) {
1932                         iaq->intr_flags_10g = INTR_RXQ;
1933                         iaq->nirq += n10g * nrxq10g;
1934 #ifdef DEV_NETMAP
1935                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1936 #endif
1937                 } else {
1938                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1939                         iaq->nirq += n10g * nofldrxq10g;
1940 #ifdef DEV_NETMAP
1941                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1942 #endif
1943                 }
1944                 if (nrxq1g >= nofldrxq1g) {
1945                         iaq->intr_flags_1g = INTR_RXQ;
1946                         iaq->nirq += n1g * nrxq1g;
1947 #ifdef DEV_NETMAP
1948                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1949 #endif
1950                 } else {
1951                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1952                         iaq->nirq += n1g * nofldrxq1g;
1953 #ifdef DEV_NETMAP
1954                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1955 #endif
1956                 }
1957                 if (iaq->nirq <= navail &&
1958                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1959                         goto allocate;
1960
1961                 /*
1962                  * Next best option: an interrupt vector for errors, one for the
1963                  * firmware event queue, and at least one per port.  At this
1964                  * point we know we'll have to downsize nrxq and/or nofldrxq
1965                  * and/or nnmrxq to fit what's available to us.
1966                  */
1967                 iaq->nirq = T4_EXTRA_INTR;
1968                 iaq->nirq += n10g + n1g;
1969                 if (iaq->nirq <= navail) {
1970                         int leftover = navail - iaq->nirq;
1971
1972                         if (n10g > 0) {
1973                                 int target = max(nrxq10g, nofldrxq10g);
1974
1975                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1976                                     INTR_RXQ : INTR_OFLD_RXQ;
1977
1978                                 n = 1;
1979                                 while (n < target && leftover >= n10g) {
1980                                         leftover -= n10g;
1981                                         iaq->nirq += n10g;
1982                                         n++;
1983                                 }
1984                                 iaq->nrxq10g = min(n, nrxq10g);
1985 #ifdef TCP_OFFLOAD
1986                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1987 #endif
1988 #ifdef DEV_NETMAP
1989                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1990 #endif
1991                         }
1992
1993                         if (n1g > 0) {
1994                                 int target = max(nrxq1g, nofldrxq1g);
1995
1996                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1997                                     INTR_RXQ : INTR_OFLD_RXQ;
1998
1999                                 n = 1;
2000                                 while (n < target && leftover >= n1g) {
2001                                         leftover -= n1g;
2002                                         iaq->nirq += n1g;
2003                                         n++;
2004                                 }
2005                                 iaq->nrxq1g = min(n, nrxq1g);
2006 #ifdef TCP_OFFLOAD
2007                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2008 #endif
2009 #ifdef DEV_NETMAP
2010                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2011 #endif
2012                         }
2013
2014                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2015                                 goto allocate;
2016                 }
2017
2018                 /*
2019                  * Least desirable option: one interrupt vector for everything.
2020                  */
2021                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2022                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2023 #ifdef TCP_OFFLOAD
2024                 if (is_offload(sc))
2025                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2026 #endif
2027 #ifdef DEV_NETMAP
2028                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2029 #endif
2030
2031 allocate:
2032                 navail = iaq->nirq;
2033                 rc = 0;
2034                 if (itype == INTR_MSIX)
2035                         rc = pci_alloc_msix(sc->dev, &navail);
2036                 else if (itype == INTR_MSI)
2037                         rc = pci_alloc_msi(sc->dev, &navail);
2038
2039                 if (rc == 0) {
2040                         if (navail == iaq->nirq)
2041                                 return (0);
2042
2043                         /*
2044                          * Didn't get the number requested.  Use whatever number
2045                          * the kernel is willing to allocate (it's in navail).
2046                          */
2047                         device_printf(sc->dev, "fewer vectors than requested, "
2048                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2049                             itype, iaq->nirq, navail);
2050                         pci_release_msi(sc->dev);
2051                         goto restart;
2052                 }
2053
2054                 device_printf(sc->dev,
2055                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2056                     itype, rc, iaq->nirq, navail);
2057         }
2058
2059         device_printf(sc->dev,
2060             "failed to find a usable interrupt type.  "
2061             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2062             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2063
2064         return (ENXIO);
2065 }
2066
2067 #define FW_VERSION(chip) ( \
2068     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2069     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2070     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2071     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2072 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2073
2074 struct fw_info {
2075         uint8_t chip;
2076         char *kld_name;
2077         char *fw_mod_name;
2078         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2079 } fw_info[] = {
2080         {
2081                 .chip = CHELSIO_T4,
2082                 .kld_name = "t4fw_cfg",
2083                 .fw_mod_name = "t4fw",
2084                 .fw_hdr = {
2085                         .chip = FW_HDR_CHIP_T4,
2086                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2087                         .intfver_nic = FW_INTFVER(T4, NIC),
2088                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2089                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2090                         .intfver_ri = FW_INTFVER(T4, RI),
2091                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2092                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2093                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2094                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2095                 },
2096         }, {
2097                 .chip = CHELSIO_T5,
2098                 .kld_name = "t5fw_cfg",
2099                 .fw_mod_name = "t5fw",
2100                 .fw_hdr = {
2101                         .chip = FW_HDR_CHIP_T5,
2102                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2103                         .intfver_nic = FW_INTFVER(T5, NIC),
2104                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2105                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2106                         .intfver_ri = FW_INTFVER(T5, RI),
2107                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2108                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2109                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2110                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2111                 },
2112         }
2113 };
2114
2115 static struct fw_info *
2116 find_fw_info(int chip)
2117 {
2118         int i;
2119
2120         for (i = 0; i < nitems(fw_info); i++) {
2121                 if (fw_info[i].chip == chip)
2122                         return (&fw_info[i]);
2123         }
2124         return (NULL);
2125 }
2126
2127 /*
2128  * Is the given firmware API compatible with the one the driver was compiled
2129  * with?
2130  */
2131 static int
2132 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2133 {
2134
2135         /* short circuit if it's the exact same firmware version */
2136         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2137                 return (1);
2138
2139         /*
2140          * XXX: Is this too conservative?  Perhaps I should limit this to the
2141          * features that are supported in the driver.
2142          */
2143 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2144         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2145             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2146             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2147                 return (1);
2148 #undef SAME_INTF
2149
2150         return (0);
2151 }
2152
2153 /*
2154  * The firmware in the KLD is usable, but should it be installed?  This routine
2155  * explains itself in detail if it indicates the KLD firmware should be
2156  * installed.
2157  */
2158 static int
2159 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2160 {
2161         const char *reason;
2162
2163         if (!card_fw_usable) {
2164                 reason = "incompatible or unusable";
2165                 goto install;
2166         }
2167
2168         if (k > c) {
2169                 reason = "older than the version bundled with this driver";
2170                 goto install;
2171         }
2172
2173         if (t4_fw_install == 2 && k != c) {
2174                 reason = "different than the version bundled with this driver";
2175                 goto install;
2176         }
2177
2178         return (0);
2179
2180 install:
2181         if (t4_fw_install == 0) {
2182                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2183                     "but the driver is prohibited from installing a different "
2184                     "firmware on the card.\n",
2185                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2186                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2187
2188                 return (0);
2189         }
2190
2191         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2192             "installing firmware %u.%u.%u.%u on card.\n",
2193             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2194             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2195             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2196             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2197
2198         return (1);
2199 }
2200 /*
2201  * Establish contact with the firmware and determine if we are the master driver
2202  * or not, and whether we are responsible for chip initialization.
2203  */
2204 static int
2205 prep_firmware(struct adapter *sc)
2206 {
2207         const struct firmware *fw = NULL, *default_cfg;
2208         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2209         enum dev_state state;
2210         struct fw_info *fw_info;
2211         struct fw_hdr *card_fw;         /* fw on the card */
2212         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2213         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2214                                            against */
2215
2216         /* Contact firmware. */
2217         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2218         if (rc < 0 || state == DEV_STATE_ERR) {
2219                 rc = -rc;
2220                 device_printf(sc->dev,
2221                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2222                 return (rc);
2223         }
2224         pf = rc;
2225         if (pf == sc->mbox)
2226                 sc->flags |= MASTER_PF;
2227         else if (state == DEV_STATE_UNINIT) {
2228                 /*
2229                  * We didn't get to be the master so we definitely won't be
2230                  * configuring the chip.  It's a bug if someone else hasn't
2231                  * configured it already.
2232                  */
2233                 device_printf(sc->dev, "couldn't be master(%d), "
2234                     "device not already initialized either(%d).\n", rc, state);
2235                 return (EDOOFUS);
2236         }
2237
2238         /* This is the firmware whose headers the driver was compiled against */
2239         fw_info = find_fw_info(chip_id(sc));
2240         if (fw_info == NULL) {
2241                 device_printf(sc->dev,
2242                     "unable to look up firmware information for chip %d.\n",
2243                     chip_id(sc));
2244                 return (EINVAL);
2245         }
2246         drv_fw = &fw_info->fw_hdr;
2247
2248         /*
2249          * The firmware KLD contains many modules.  The KLD name is also the
2250          * name of the module that contains the default config file.
2251          */
2252         default_cfg = firmware_get(fw_info->kld_name);
2253
2254         /* Read the header of the firmware on the card */
2255         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2256         rc = -t4_read_flash(sc, FLASH_FW_START,
2257             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2258         if (rc == 0)
2259                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2260         else {
2261                 device_printf(sc->dev,
2262                     "Unable to read card's firmware header: %d\n", rc);
2263                 card_fw_usable = 0;
2264         }
2265
2266         /* This is the firmware in the KLD */
2267         fw = firmware_get(fw_info->fw_mod_name);
2268         if (fw != NULL) {
2269                 kld_fw = (const void *)fw->data;
2270                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2271         } else {
2272                 kld_fw = NULL;
2273                 kld_fw_usable = 0;
2274         }
2275
2276         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2277             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2278                 /*
2279                  * Common case: the firmware on the card is an exact match and
2280                  * the KLD is an exact match too, or the KLD is
2281                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2282                  * here -- use cxgbetool loadfw if you want to reinstall the
2283                  * same firmware as the one on the card.
2284                  */
2285         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2286             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2287             be32toh(card_fw->fw_ver))) {
2288
2289                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2290                 if (rc != 0) {
2291                         device_printf(sc->dev,
2292                             "failed to install firmware: %d\n", rc);
2293                         goto done;
2294                 }
2295
2296                 /* Installed successfully, update the cached header too. */
2297                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2298                 card_fw_usable = 1;
2299                 need_fw_reset = 0;      /* already reset as part of load_fw */
2300         }
2301
2302         if (!card_fw_usable) {
2303                 uint32_t d, c, k;
2304
2305                 d = ntohl(drv_fw->fw_ver);
2306                 c = ntohl(card_fw->fw_ver);
2307                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2308
2309                 device_printf(sc->dev, "Cannot find a usable firmware: "
2310                     "fw_install %d, chip state %d, "
2311                     "driver compiled with %d.%d.%d.%d, "
2312                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2313                     t4_fw_install, state,
2314                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2315                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2316                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2317                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2318                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2319                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2320                 rc = EINVAL;
2321                 goto done;
2322         }
2323
2324         /* We're using whatever's on the card and it's known to be good. */
2325         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2326         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2327             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2328             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2329             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2330             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2331         t4_get_tp_version(sc, &sc->params.tp_vers);
2332
2333         /* Reset device */
2334         if (need_fw_reset &&
2335             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2336                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2337                 if (rc != ETIMEDOUT && rc != EIO)
2338                         t4_fw_bye(sc, sc->mbox);
2339                 goto done;
2340         }
2341         sc->flags |= FW_OK;
2342
2343         rc = get_params__pre_init(sc);
2344         if (rc != 0)
2345                 goto done; /* error message displayed already */
2346
2347         /* Partition adapter resources as specified in the config file. */
2348         if (state == DEV_STATE_UNINIT) {
2349
2350                 KASSERT(sc->flags & MASTER_PF,
2351                     ("%s: trying to change chip settings when not master.",
2352                     __func__));
2353
2354                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2355                 if (rc != 0)
2356                         goto done;      /* error message displayed already */
2357
2358                 t4_tweak_chip_settings(sc);
2359
2360                 /* get basic stuff going */
2361                 rc = -t4_fw_initialize(sc, sc->mbox);
2362                 if (rc != 0) {
2363                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2364                         goto done;
2365                 }
2366         } else {
2367                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2368                 sc->cfcsum = 0;
2369         }
2370
2371 done:
2372         free(card_fw, M_CXGBE);
2373         if (fw != NULL)
2374                 firmware_put(fw, FIRMWARE_UNLOAD);
2375         if (default_cfg != NULL)
2376                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2377
2378         return (rc);
2379 }
2380
2381 #define FW_PARAM_DEV(param) \
2382         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2383          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2384 #define FW_PARAM_PFVF(param) \
2385         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2386          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2387
2388 /*
2389  * Partition chip resources for use between various PFs, VFs, etc.
2390  */
2391 static int
2392 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2393     const char *name_prefix)
2394 {
2395         const struct firmware *cfg = NULL;
2396         int rc = 0;
2397         struct fw_caps_config_cmd caps;
2398         uint32_t mtype, moff, finicsum, cfcsum;
2399
2400         /*
2401          * Figure out what configuration file to use.  Pick the default config
2402          * file for the card if the user hasn't specified one explicitly.
2403          */
2404         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2405         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2406                 /* Card specific overrides go here. */
2407                 if (pci_get_device(sc->dev) == 0x440a)
2408                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2409                 if (is_fpga(sc))
2410                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2411         }
2412
2413         /*
2414          * We need to load another module if the profile is anything except
2415          * "default" or "flash".
2416          */
2417         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2418             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2419                 char s[32];
2420
2421                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2422                 cfg = firmware_get(s);
2423                 if (cfg == NULL) {
2424                         if (default_cfg != NULL) {
2425                                 device_printf(sc->dev,
2426                                     "unable to load module \"%s\" for "
2427                                     "configuration profile \"%s\", will use "
2428                                     "the default config file instead.\n",
2429                                     s, sc->cfg_file);
2430                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2431                                     "%s", DEFAULT_CF);
2432                         } else {
2433                                 device_printf(sc->dev,
2434                                     "unable to load module \"%s\" for "
2435                                     "configuration profile \"%s\", will use "
2436                                     "the config file on the card's flash "
2437                                     "instead.\n", s, sc->cfg_file);
2438                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2439                                     "%s", FLASH_CF);
2440                         }
2441                 }
2442         }
2443
2444         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2445             default_cfg == NULL) {
2446                 device_printf(sc->dev,
2447                     "default config file not available, will use the config "
2448                     "file on the card's flash instead.\n");
2449                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2450         }
2451
2452         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2453                 u_int cflen, i, n;
2454                 const uint32_t *cfdata;
2455                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2456
2457                 KASSERT(cfg != NULL || default_cfg != NULL,
2458                     ("%s: no config to upload", __func__));
2459
2460                 /*
2461                  * Ask the firmware where it wants us to upload the config file.
2462                  */
2463                 param = FW_PARAM_DEV(CF);
2464                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2465                 if (rc != 0) {
2466                         /* No support for config file?  Shouldn't happen. */
2467                         device_printf(sc->dev,
2468                             "failed to query config file location: %d.\n", rc);
2469                         goto done;
2470                 }
2471                 mtype = G_FW_PARAMS_PARAM_Y(val);
2472                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2473
2474                 /*
2475                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2476                  * useless stuffing/comments at the end of the config file so
2477                  * it's ok to simply throw away the last remaining bytes when
2478                  * the config file is not an exact multiple of 4.  This also
2479                  * helps with the validate_mt_off_len check.
2480                  */
2481                 if (cfg != NULL) {
2482                         cflen = cfg->datasize & ~3;
2483                         cfdata = cfg->data;
2484                 } else {
2485                         cflen = default_cfg->datasize & ~3;
2486                         cfdata = default_cfg->data;
2487                 }
2488
2489                 if (cflen > FLASH_CFG_MAX_SIZE) {
2490                         device_printf(sc->dev,
2491                             "config file too long (%d, max allowed is %d).  "
2492                             "Will try to use the config on the card, if any.\n",
2493                             cflen, FLASH_CFG_MAX_SIZE);
2494                         goto use_config_on_flash;
2495                 }
2496
2497                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2498                 if (rc != 0) {
2499                         device_printf(sc->dev,
2500                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2501                             "Will try to use the config on the card, if any.\n",
2502                             __func__, mtype, moff, cflen, rc);
2503                         goto use_config_on_flash;
2504                 }
2505
2506                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2507                 while (cflen) {
2508                         off = position_memwin(sc, 2, addr);
2509                         n = min(cflen, mw_aperture - off);
2510                         for (i = 0; i < n; i += 4)
2511                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2512                         cflen -= n;
2513                         addr += n;
2514                 }
2515         } else {
2516 use_config_on_flash:
2517                 mtype = FW_MEMTYPE_FLASH;
2518                 moff = t4_flash_cfg_addr(sc);
2519         }
2520
2521         bzero(&caps, sizeof(caps));
2522         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2523             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2524         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2525             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2526             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2527         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2528         if (rc != 0) {
2529                 device_printf(sc->dev,
2530                     "failed to pre-process config file: %d "
2531                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2532                 goto done;
2533         }
2534
2535         finicsum = be32toh(caps.finicsum);
2536         cfcsum = be32toh(caps.cfcsum);
2537         if (finicsum != cfcsum) {
2538                 device_printf(sc->dev,
2539                     "WARNING: config file checksum mismatch: %08x %08x\n",
2540                     finicsum, cfcsum);
2541         }
2542         sc->cfcsum = cfcsum;
2543
2544 #define LIMIT_CAPS(x) do { \
2545         caps.x &= htobe16(t4_##x##_allowed); \
2546 } while (0)
2547
2548         /*
2549          * Let the firmware know what features will (not) be used so it can tune
2550          * things accordingly.
2551          */
2552         LIMIT_CAPS(linkcaps);
2553         LIMIT_CAPS(niccaps);
2554         LIMIT_CAPS(toecaps);
2555         LIMIT_CAPS(rdmacaps);
2556         LIMIT_CAPS(iscsicaps);
2557         LIMIT_CAPS(fcoecaps);
2558 #undef LIMIT_CAPS
2559
2560         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2561             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2562         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2563         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2564         if (rc != 0) {
2565                 device_printf(sc->dev,
2566                     "failed to process config file: %d.\n", rc);
2567         }
2568 done:
2569         if (cfg != NULL)
2570                 firmware_put(cfg, FIRMWARE_UNLOAD);
2571         return (rc);
2572 }
2573
2574 /*
2575  * Retrieve parameters that are needed (or nice to have) very early.
2576  */
2577 static int
2578 get_params__pre_init(struct adapter *sc)
2579 {
2580         int rc;
2581         uint32_t param[2], val[2];
2582         struct fw_devlog_cmd cmd;
2583         struct devlog_params *dlog = &sc->params.devlog;
2584
2585         param[0] = FW_PARAM_DEV(PORTVEC);
2586         param[1] = FW_PARAM_DEV(CCLK);
2587         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2588         if (rc != 0) {
2589                 device_printf(sc->dev,
2590                     "failed to query parameters (pre_init): %d.\n", rc);
2591                 return (rc);
2592         }
2593
2594         sc->params.portvec = val[0];
2595         sc->params.nports = bitcount32(val[0]);
2596         sc->params.vpd.cclk = val[1];
2597
2598         /* Read device log parameters. */
2599         bzero(&cmd, sizeof(cmd));
2600         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2601             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2602         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2603         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2604         if (rc != 0) {
2605                 device_printf(sc->dev,
2606                     "failed to get devlog parameters: %d.\n", rc);
2607                 bzero(dlog, sizeof (*dlog));
2608                 rc = 0; /* devlog isn't critical for device operation */
2609         } else {
2610                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2611                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2612                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2613                 dlog->size = be32toh(cmd.memsize_devlog);
2614         }
2615
2616         return (rc);
2617 }
2618
2619 /*
2620  * Retrieve various parameters that are of interest to the driver.  The device
2621  * has been initialized by the firmware at this point.
2622  */
2623 static int
2624 get_params__post_init(struct adapter *sc)
2625 {
2626         int rc;
2627         uint32_t param[7], val[7];
2628         struct fw_caps_config_cmd caps;
2629
2630         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2631         param[1] = FW_PARAM_PFVF(EQ_START);
2632         param[2] = FW_PARAM_PFVF(FILTER_START);
2633         param[3] = FW_PARAM_PFVF(FILTER_END);
2634         param[4] = FW_PARAM_PFVF(L2T_START);
2635         param[5] = FW_PARAM_PFVF(L2T_END);
2636         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2637         if (rc != 0) {
2638                 device_printf(sc->dev,
2639                     "failed to query parameters (post_init): %d.\n", rc);
2640                 return (rc);
2641         }
2642
2643         sc->sge.iq_start = val[0];
2644         sc->sge.eq_start = val[1];
2645         sc->tids.ftid_base = val[2];
2646         sc->tids.nftids = val[3] - val[2] + 1;
2647         sc->params.ftid_min = val[2];
2648         sc->params.ftid_max = val[3];
2649         sc->vres.l2t.start = val[4];
2650         sc->vres.l2t.size = val[5] - val[4] + 1;
2651         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2652             ("%s: L2 table size (%u) larger than expected (%u)",
2653             __func__, sc->vres.l2t.size, L2T_SIZE));
2654
2655         /* get capabilites */
2656         bzero(&caps, sizeof(caps));
2657         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2658             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2659         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2660         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2661         if (rc != 0) {
2662                 device_printf(sc->dev,
2663                     "failed to get card capabilities: %d.\n", rc);
2664                 return (rc);
2665         }
2666
2667 #define READ_CAPS(x) do { \
2668         sc->x = htobe16(caps.x); \
2669 } while (0)
2670         READ_CAPS(linkcaps);
2671         READ_CAPS(niccaps);
2672         READ_CAPS(toecaps);
2673         READ_CAPS(rdmacaps);
2674         READ_CAPS(iscsicaps);
2675         READ_CAPS(fcoecaps);
2676
2677         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2678                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2679                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2680                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2681                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2682                 if (rc != 0) {
2683                         device_printf(sc->dev,
2684                             "failed to query NIC parameters: %d.\n", rc);
2685                         return (rc);
2686                 }
2687                 sc->tids.etid_base = val[0];
2688                 sc->params.etid_min = val[0];
2689                 sc->tids.netids = val[1] - val[0] + 1;
2690                 sc->params.netids = sc->tids.netids;
2691                 sc->params.eo_wr_cred = val[2];
2692                 sc->params.ethoffload = 1;
2693         }
2694
2695         if (sc->toecaps) {
2696                 /* query offload-related parameters */
2697                 param[0] = FW_PARAM_DEV(NTID);
2698                 param[1] = FW_PARAM_PFVF(SERVER_START);
2699                 param[2] = FW_PARAM_PFVF(SERVER_END);
2700                 param[3] = FW_PARAM_PFVF(TDDP_START);
2701                 param[4] = FW_PARAM_PFVF(TDDP_END);
2702                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2703                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2704                 if (rc != 0) {
2705                         device_printf(sc->dev,
2706                             "failed to query TOE parameters: %d.\n", rc);
2707                         return (rc);
2708                 }
2709                 sc->tids.ntids = val[0];
2710                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2711                 sc->tids.stid_base = val[1];
2712                 sc->tids.nstids = val[2] - val[1] + 1;
2713                 sc->vres.ddp.start = val[3];
2714                 sc->vres.ddp.size = val[4] - val[3] + 1;
2715                 sc->params.ofldq_wr_cred = val[5];
2716                 sc->params.offload = 1;
2717         }
2718         if (sc->rdmacaps) {
2719                 param[0] = FW_PARAM_PFVF(STAG_START);
2720                 param[1] = FW_PARAM_PFVF(STAG_END);
2721                 param[2] = FW_PARAM_PFVF(RQ_START);
2722                 param[3] = FW_PARAM_PFVF(RQ_END);
2723                 param[4] = FW_PARAM_PFVF(PBL_START);
2724                 param[5] = FW_PARAM_PFVF(PBL_END);
2725                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2726                 if (rc != 0) {
2727                         device_printf(sc->dev,
2728                             "failed to query RDMA parameters(1): %d.\n", rc);
2729                         return (rc);
2730                 }
2731                 sc->vres.stag.start = val[0];
2732                 sc->vres.stag.size = val[1] - val[0] + 1;
2733                 sc->vres.rq.start = val[2];
2734                 sc->vres.rq.size = val[3] - val[2] + 1;
2735                 sc->vres.pbl.start = val[4];
2736                 sc->vres.pbl.size = val[5] - val[4] + 1;
2737
2738                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2739                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2740                 param[2] = FW_PARAM_PFVF(CQ_START);
2741                 param[3] = FW_PARAM_PFVF(CQ_END);
2742                 param[4] = FW_PARAM_PFVF(OCQ_START);
2743                 param[5] = FW_PARAM_PFVF(OCQ_END);
2744                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2745                 if (rc != 0) {
2746                         device_printf(sc->dev,
2747                             "failed to query RDMA parameters(2): %d.\n", rc);
2748                         return (rc);
2749                 }
2750                 sc->vres.qp.start = val[0];
2751                 sc->vres.qp.size = val[1] - val[0] + 1;
2752                 sc->vres.cq.start = val[2];
2753                 sc->vres.cq.size = val[3] - val[2] + 1;
2754                 sc->vres.ocq.start = val[4];
2755                 sc->vres.ocq.size = val[5] - val[4] + 1;
2756         }
2757         if (sc->iscsicaps) {
2758                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2759                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2760                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2761                 if (rc != 0) {
2762                         device_printf(sc->dev,
2763                             "failed to query iSCSI parameters: %d.\n", rc);
2764                         return (rc);
2765                 }
2766                 sc->vres.iscsi.start = val[0];
2767                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2768         }
2769
2770         /*
2771          * We've got the params we wanted to query via the firmware.  Now grab
2772          * some others directly from the chip.
2773          */
2774         rc = t4_read_chip_settings(sc);
2775
2776         return (rc);
2777 }
2778
2779 static int
2780 set_params__post_init(struct adapter *sc)
2781 {
2782         uint32_t param, val;
2783
2784         /* ask for encapsulated CPLs */
2785         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2786         val = 1;
2787         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2788
2789         return (0);
2790 }
2791
2792 #undef FW_PARAM_PFVF
2793 #undef FW_PARAM_DEV
2794
2795 static void
2796 t4_set_desc(struct adapter *sc)
2797 {
2798         char buf[128];
2799         struct adapter_params *p = &sc->params;
2800
2801         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2802             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2803             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2804
2805         device_set_desc_copy(sc->dev, buf);
2806 }
2807
2808 static void
2809 build_medialist(struct port_info *pi, struct ifmedia *media)
2810 {
2811         int data, m;
2812
2813         PORT_LOCK(pi);
2814
2815         ifmedia_removeall(media);
2816
2817         m = IFM_ETHER | IFM_FDX;
2818         data = (pi->port_type << 8) | pi->mod_type;
2819
2820         switch(pi->port_type) {
2821         case FW_PORT_TYPE_BT_XFI:
2822         case FW_PORT_TYPE_BT_XAUI:
2823                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2824                 /* fall through */
2825
2826         case FW_PORT_TYPE_BT_SGMII:
2827                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2828                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2829                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2830                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2831                 break;
2832
2833         case FW_PORT_TYPE_CX4:
2834                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2835                 ifmedia_set(media, m | IFM_10G_CX4);
2836                 break;
2837
2838         case FW_PORT_TYPE_QSFP_10G:
2839         case FW_PORT_TYPE_SFP:
2840         case FW_PORT_TYPE_FIBER_XFI:
2841         case FW_PORT_TYPE_FIBER_XAUI:
2842                 switch (pi->mod_type) {
2843
2844                 case FW_PORT_MOD_TYPE_LR:
2845                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2846                         ifmedia_set(media, m | IFM_10G_LR);
2847                         break;
2848
2849                 case FW_PORT_MOD_TYPE_SR:
2850                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2851                         ifmedia_set(media, m | IFM_10G_SR);
2852                         break;
2853
2854                 case FW_PORT_MOD_TYPE_LRM:
2855                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2856                         ifmedia_set(media, m | IFM_10G_LRM);
2857                         break;
2858
2859                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2860                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2861                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2862                         ifmedia_set(media, m | IFM_10G_TWINAX);
2863                         break;
2864
2865                 case FW_PORT_MOD_TYPE_NONE:
2866                         m &= ~IFM_FDX;
2867                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2868                         ifmedia_set(media, m | IFM_NONE);
2869                         break;
2870
2871                 case FW_PORT_MOD_TYPE_NA:
2872                 case FW_PORT_MOD_TYPE_ER:
2873                 default:
2874                         device_printf(pi->dev,
2875                             "unknown port_type (%d), mod_type (%d)\n",
2876                             pi->port_type, pi->mod_type);
2877                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2878                         ifmedia_set(media, m | IFM_UNKNOWN);
2879                         break;
2880                 }
2881                 break;
2882
2883         case FW_PORT_TYPE_QSFP:
2884                 switch (pi->mod_type) {
2885
2886                 case FW_PORT_MOD_TYPE_LR:
2887                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2888                         ifmedia_set(media, m | IFM_40G_LR4);
2889                         break;
2890
2891                 case FW_PORT_MOD_TYPE_SR:
2892                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2893                         ifmedia_set(media, m | IFM_40G_SR4);
2894                         break;
2895
2896                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2897                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2898                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2899                         ifmedia_set(media, m | IFM_40G_CR4);
2900                         break;
2901
2902                 case FW_PORT_MOD_TYPE_NONE:
2903                         m &= ~IFM_FDX;
2904                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2905                         ifmedia_set(media, m | IFM_NONE);
2906                         break;
2907
2908                 default:
2909                         device_printf(pi->dev,
2910                             "unknown port_type (%d), mod_type (%d)\n",
2911                             pi->port_type, pi->mod_type);
2912                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2913                         ifmedia_set(media, m | IFM_UNKNOWN);
2914                         break;
2915                 }
2916                 break;
2917
2918         default:
2919                 device_printf(pi->dev,
2920                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2921                     pi->mod_type);
2922                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2923                 ifmedia_set(media, m | IFM_UNKNOWN);
2924                 break;
2925         }
2926
2927         PORT_UNLOCK(pi);
2928 }
2929
2930 #define FW_MAC_EXACT_CHUNK      7
2931
2932 /*
2933  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2934  * indicates which parameters should be programmed (the rest are left alone).
2935  */
2936 int
2937 update_mac_settings(struct ifnet *ifp, int flags)
2938 {
2939         int rc = 0;
2940         struct port_info *pi = ifp->if_softc;
2941         struct adapter *sc = pi->adapter;
2942         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2943         uint16_t viid = 0xffff;
2944         int16_t *xact_addr_filt = NULL;
2945
2946         ASSERT_SYNCHRONIZED_OP(sc);
2947         KASSERT(flags, ("%s: not told what to update.", __func__));
2948
2949         if (ifp == pi->ifp) {
2950                 viid = pi->viid;
2951                 xact_addr_filt = &pi->xact_addr_filt;
2952         }
2953 #ifdef DEV_NETMAP
2954         else if (ifp == pi->nm_ifp) {
2955                 viid = pi->nm_viid;
2956                 xact_addr_filt = &pi->nm_xact_addr_filt;
2957         }
2958 #endif
2959         if (flags & XGMAC_MTU)
2960                 mtu = ifp->if_mtu;
2961
2962         if (flags & XGMAC_PROMISC)
2963                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2964
2965         if (flags & XGMAC_ALLMULTI)
2966                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2967
2968         if (flags & XGMAC_VLANEX)
2969                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2970
2971         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2972                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2973                     1, vlanex, false);
2974                 if (rc) {
2975                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2976                             rc);
2977                         return (rc);
2978                 }
2979         }
2980
2981         if (flags & XGMAC_UCADDR) {
2982                 uint8_t ucaddr[ETHER_ADDR_LEN];
2983
2984                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2985                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2986                     true, true);
2987                 if (rc < 0) {
2988                         rc = -rc;
2989                         if_printf(ifp, "change_mac failed: %d\n", rc);
2990                         return (rc);
2991                 } else {
2992                         *xact_addr_filt = rc;
2993                         rc = 0;
2994                 }
2995         }
2996
2997         if (flags & XGMAC_MCADDRS) {
2998                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2999                 int del = 1;
3000                 uint64_t hash = 0;
3001                 struct ifmultiaddr *ifma;
3002                 int i = 0, j;
3003
3004                 if_maddr_rlock(ifp);
3005                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3006                         if (ifma->ifma_addr->sa_family != AF_LINK)
3007                                 continue;
3008                         mcaddr[i] =
3009                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3010                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3011                         i++;
3012
3013                         if (i == FW_MAC_EXACT_CHUNK) {
3014                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3015                                     i, mcaddr, NULL, &hash, 0);
3016                                 if (rc < 0) {
3017                                         rc = -rc;
3018                                         for (j = 0; j < i; j++) {
3019                                                 if_printf(ifp,
3020                                                     "failed to add mc address"
3021                                                     " %02x:%02x:%02x:"
3022                                                     "%02x:%02x:%02x rc=%d\n",
3023                                                     mcaddr[j][0], mcaddr[j][1],
3024                                                     mcaddr[j][2], mcaddr[j][3],
3025                                                     mcaddr[j][4], mcaddr[j][5],
3026                                                     rc);
3027                                         }
3028                                         goto mcfail;
3029                                 }
3030                                 del = 0;
3031                                 i = 0;
3032                         }
3033                 }
3034                 if (i > 0) {
3035                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3036                             mcaddr, NULL, &hash, 0);
3037                         if (rc < 0) {
3038                                 rc = -rc;
3039                                 for (j = 0; j < i; j++) {
3040                                         if_printf(ifp,
3041                                             "failed to add mc address"
3042                                             " %02x:%02x:%02x:"
3043                                             "%02x:%02x:%02x rc=%d\n",
3044                                             mcaddr[j][0], mcaddr[j][1],
3045                                             mcaddr[j][2], mcaddr[j][3],
3046                                             mcaddr[j][4], mcaddr[j][5],
3047                                             rc);
3048                                 }
3049                                 goto mcfail;
3050                         }
3051                 }
3052
3053                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3054                 if (rc != 0)
3055                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3056 mcfail:
3057                 if_maddr_runlock(ifp);
3058         }
3059
3060         return (rc);
3061 }
3062
3063 /*
3064  * {begin|end}_synchronized_op must be called from the same thread.
3065  */
3066 int
3067 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3068     char *wmesg)
3069 {
3070         int rc, pri;
3071
3072 #ifdef WITNESS
3073         /* the caller thinks it's ok to sleep, but is it really? */
3074         if (flags & SLEEP_OK)
3075                 pause("t4slptst", 1);
3076 #endif
3077
3078         if (INTR_OK)
3079                 pri = PCATCH;
3080         else
3081                 pri = 0;
3082
3083         ADAPTER_LOCK(sc);
3084         for (;;) {
3085
3086                 if (pi && IS_DOOMED(pi)) {
3087                         rc = ENXIO;
3088                         goto done;
3089                 }
3090
3091                 if (!IS_BUSY(sc)) {
3092                         rc = 0;
3093                         break;
3094                 }
3095
3096                 if (!(flags & SLEEP_OK)) {
3097                         rc = EBUSY;
3098                         goto done;
3099                 }
3100
3101                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3102                         rc = EINTR;
3103                         goto done;
3104                 }
3105         }
3106
3107         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3108         SET_BUSY(sc);
3109 #ifdef INVARIANTS
3110         sc->last_op = wmesg;
3111         sc->last_op_thr = curthread;
3112 #endif
3113
3114 done:
3115         if (!(flags & HOLD_LOCK) || rc)
3116                 ADAPTER_UNLOCK(sc);
3117
3118         return (rc);
3119 }
3120
3121 /*
3122  * {begin|end}_synchronized_op must be called from the same thread.
3123  */
3124 void
3125 end_synchronized_op(struct adapter *sc, int flags)
3126 {
3127
3128         if (flags & LOCK_HELD)
3129                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3130         else
3131                 ADAPTER_LOCK(sc);
3132
3133         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3134         CLR_BUSY(sc);
3135         wakeup(&sc->flags);
3136         ADAPTER_UNLOCK(sc);
3137 }
3138
3139 static int
3140 cxgbe_init_synchronized(struct port_info *pi)
3141 {
3142         struct adapter *sc = pi->adapter;
3143         struct ifnet *ifp = pi->ifp;
3144         int rc = 0, i;
3145         struct sge_txq *txq;
3146
3147         ASSERT_SYNCHRONIZED_OP(sc);
3148
3149         if (isset(&sc->open_device_map, pi->port_id)) {
3150                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3151                     ("mismatch between open_device_map and if_drv_flags"));
3152                 return (0);     /* already running */
3153         }
3154
3155         if (!(sc->flags & FULL_INIT_DONE) &&
3156             ((rc = adapter_full_init(sc)) != 0))
3157                 return (rc);    /* error message displayed already */
3158
3159         if (!(pi->flags & PORT_INIT_DONE) &&
3160             ((rc = port_full_init(pi)) != 0))
3161                 return (rc); /* error message displayed already */
3162
3163         rc = update_mac_settings(ifp, XGMAC_ALL);
3164         if (rc)
3165                 goto done;      /* error message displayed already */
3166
3167         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3168         if (rc != 0) {
3169                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3170                 goto done;
3171         }
3172
3173         /*
3174          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3175          * if this changes.
3176          */
3177
3178         for_each_txq(pi, i, txq) {
3179                 TXQ_LOCK(txq);
3180                 txq->eq.flags |= EQ_ENABLED;
3181                 TXQ_UNLOCK(txq);
3182         }
3183
3184         /*
3185          * The first iq of the first port to come up is used for tracing.
3186          */
3187         if (sc->traceq < 0) {
3188                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3189                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3190                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3191                     V_QUEUENUMBER(sc->traceq));
3192                 pi->flags |= HAS_TRACEQ;
3193         }
3194
3195         /* all ok */
3196         setbit(&sc->open_device_map, pi->port_id);
3197         PORT_LOCK(pi);
3198         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3199         PORT_UNLOCK(pi);
3200
3201         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3202 done:
3203         if (rc != 0)
3204                 cxgbe_uninit_synchronized(pi);
3205
3206         return (rc);
3207 }
3208
3209 /*
3210  * Idempotent.
3211  */
3212 static int
3213 cxgbe_uninit_synchronized(struct port_info *pi)
3214 {
3215         struct adapter *sc = pi->adapter;
3216         struct ifnet *ifp = pi->ifp;
3217         int rc, i;
3218         struct sge_txq *txq;
3219
3220         ASSERT_SYNCHRONIZED_OP(sc);
3221
3222         /*
3223          * Disable the VI so that all its data in either direction is discarded
3224          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3225          * tick) intact as the TP can deliver negative advice or data that it's
3226          * holding in its RAM (for an offloaded connection) even after the VI is
3227          * disabled.
3228          */
3229         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3230         if (rc) {
3231                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3232                 return (rc);
3233         }
3234
3235         for_each_txq(pi, i, txq) {
3236                 TXQ_LOCK(txq);
3237                 txq->eq.flags &= ~EQ_ENABLED;
3238                 TXQ_UNLOCK(txq);
3239         }
3240
3241         clrbit(&sc->open_device_map, pi->port_id);
3242         PORT_LOCK(pi);
3243         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3244         PORT_UNLOCK(pi);
3245
3246         pi->link_cfg.link_ok = 0;
3247         pi->link_cfg.speed = 0;
3248         pi->linkdnrc = -1;
3249         t4_os_link_changed(sc, pi->port_id, 0, -1);
3250
3251         return (0);
3252 }
3253
3254 /*
3255  * It is ok for this function to fail midway and return right away.  t4_detach
3256  * will walk the entire sc->irq list and clean up whatever is valid.
3257  */
3258 static int
3259 setup_intr_handlers(struct adapter *sc)
3260 {
3261         int rc, rid, p, q;
3262         char s[8];
3263         struct irq *irq;
3264         struct port_info *pi;
3265         struct sge_rxq *rxq;
3266 #ifdef TCP_OFFLOAD
3267         struct sge_ofld_rxq *ofld_rxq;
3268 #endif
3269 #ifdef DEV_NETMAP
3270         struct sge_nm_rxq *nm_rxq;
3271 #endif
3272
3273         /*
3274          * Setup interrupts.
3275          */
3276         irq = &sc->irq[0];
3277         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3278         if (sc->intr_count == 1)
3279                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3280
3281         /* Multiple interrupts. */
3282         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3283             ("%s: too few intr.", __func__));
3284
3285         /* The first one is always error intr */
3286         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3287         if (rc != 0)
3288                 return (rc);
3289         irq++;
3290         rid++;
3291
3292         /* The second one is always the firmware event queue */
3293         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3294         if (rc != 0)
3295                 return (rc);
3296         irq++;
3297         rid++;
3298
3299         for_each_port(sc, p) {
3300                 pi = sc->port[p];
3301
3302                 if (pi->flags & INTR_RXQ) {
3303                         for_each_rxq(pi, q, rxq) {
3304                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3305                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3306                                     s);
3307                                 if (rc != 0)
3308                                         return (rc);
3309                                 irq++;
3310                                 rid++;
3311                         }
3312                 }
3313 #ifdef TCP_OFFLOAD
3314                 if (pi->flags & INTR_OFLD_RXQ) {
3315                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3316                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3317                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3318                                     ofld_rxq, s);
3319                                 if (rc != 0)
3320                                         return (rc);
3321                                 irq++;
3322                                 rid++;
3323                         }
3324                 }
3325 #endif
3326 #ifdef DEV_NETMAP
3327                 if (pi->flags & INTR_NM_RXQ) {
3328                         for_each_nm_rxq(pi, q, nm_rxq) {
3329                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3330                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3331                                     nm_rxq, s);
3332                                 if (rc != 0)
3333                                         return (rc);
3334                                 irq++;
3335                                 rid++;
3336                         }
3337                 }
3338 #endif
3339         }
3340         MPASS(irq == &sc->irq[sc->intr_count]);
3341
3342         return (0);
3343 }
3344
3345 int
3346 adapter_full_init(struct adapter *sc)
3347 {
3348         int rc, i;
3349
3350         ASSERT_SYNCHRONIZED_OP(sc);
3351         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3352         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3353             ("%s: FULL_INIT_DONE already", __func__));
3354
3355         /*
3356          * queues that belong to the adapter (not any particular port).
3357          */
3358         rc = t4_setup_adapter_queues(sc);
3359         if (rc != 0)
3360                 goto done;
3361
3362         for (i = 0; i < nitems(sc->tq); i++) {
3363                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3364                     taskqueue_thread_enqueue, &sc->tq[i]);
3365                 if (sc->tq[i] == NULL) {
3366                         device_printf(sc->dev,
3367                             "failed to allocate task queue %d\n", i);
3368                         rc = ENOMEM;
3369                         goto done;
3370                 }
3371                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3372                     device_get_nameunit(sc->dev), i);
3373         }
3374
3375         t4_intr_enable(sc);
3376         sc->flags |= FULL_INIT_DONE;
3377 done:
3378         if (rc != 0)
3379                 adapter_full_uninit(sc);
3380
3381         return (rc);
3382 }
3383
3384 int
3385 adapter_full_uninit(struct adapter *sc)
3386 {
3387         int i;
3388
3389         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3390
3391         t4_teardown_adapter_queues(sc);
3392
3393         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3394                 taskqueue_free(sc->tq[i]);
3395                 sc->tq[i] = NULL;
3396         }
3397
3398         sc->flags &= ~FULL_INIT_DONE;
3399
3400         return (0);
3401 }
3402
3403 int
3404 port_full_init(struct port_info *pi)
3405 {
3406         struct adapter *sc = pi->adapter;
3407         struct ifnet *ifp = pi->ifp;
3408         uint16_t *rss;
3409         struct sge_rxq *rxq;
3410         int rc, i, j;
3411
3412         ASSERT_SYNCHRONIZED_OP(sc);
3413         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3414             ("%s: PORT_INIT_DONE already", __func__));
3415
3416         sysctl_ctx_init(&pi->ctx);
3417         pi->flags |= PORT_SYSCTL_CTX;
3418
3419         /*
3420          * Allocate tx/rx/fl queues for this port.
3421          */
3422         rc = t4_setup_port_queues(pi);
3423         if (rc != 0)
3424                 goto done;      /* error message displayed already */
3425
3426         /*
3427          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3428          */
3429         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3430         for (i = 0; i < pi->rss_size;) {
3431                 for_each_rxq(pi, j, rxq) {
3432                         rss[i++] = rxq->iq.abs_id;
3433                         if (i == pi->rss_size)
3434                                 break;
3435                 }
3436         }
3437
3438         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3439             pi->rss_size);
3440         if (rc != 0) {
3441                 if_printf(ifp, "rss_config failed: %d\n", rc);
3442                 goto done;
3443         }
3444
3445         pi->rss = rss;
3446         pi->flags |= PORT_INIT_DONE;
3447 done:
3448         if (rc != 0)
3449                 port_full_uninit(pi);
3450
3451         return (rc);
3452 }
3453
3454 /*
3455  * Idempotent.
3456  */
3457 int
3458 port_full_uninit(struct port_info *pi)
3459 {
3460         struct adapter *sc = pi->adapter;
3461         int i;
3462         struct sge_rxq *rxq;
3463         struct sge_txq *txq;
3464 #ifdef TCP_OFFLOAD
3465         struct sge_ofld_rxq *ofld_rxq;
3466         struct sge_wrq *ofld_txq;
3467 #endif
3468
3469         if (pi->flags & PORT_INIT_DONE) {
3470
3471                 /* Need to quiesce queues.  */
3472
3473                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3474
3475                 for_each_txq(pi, i, txq) {
3476                         quiesce_txq(sc, txq);
3477                 }
3478
3479 #ifdef TCP_OFFLOAD
3480                 for_each_ofld_txq(pi, i, ofld_txq) {
3481                         quiesce_wrq(sc, ofld_txq);
3482                 }
3483 #endif
3484
3485                 for_each_rxq(pi, i, rxq) {
3486                         quiesce_iq(sc, &rxq->iq);
3487                         quiesce_fl(sc, &rxq->fl);
3488                 }
3489
3490 #ifdef TCP_OFFLOAD
3491                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3492                         quiesce_iq(sc, &ofld_rxq->iq);
3493                         quiesce_fl(sc, &ofld_rxq->fl);
3494                 }
3495 #endif
3496                 free(pi->rss, M_CXGBE);
3497         }
3498
3499         t4_teardown_port_queues(pi);
3500         pi->flags &= ~PORT_INIT_DONE;
3501
3502         return (0);
3503 }
3504
3505 static void
3506 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3507 {
3508         struct sge_eq *eq = &txq->eq;
3509         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3510
3511         (void) sc;      /* unused */
3512
3513 #ifdef INVARIANTS
3514         TXQ_LOCK(txq);
3515         MPASS((eq->flags & EQ_ENABLED) == 0);
3516         TXQ_UNLOCK(txq);
3517 #endif
3518
3519         /* Wait for the mp_ring to empty. */
3520         while (!mp_ring_is_idle(txq->r)) {
3521                 mp_ring_check_drainage(txq->r, 0);
3522                 pause("rquiesce", 1);
3523         }
3524
3525         /* Then wait for the hardware to finish. */
3526         while (spg->cidx != htobe16(eq->pidx))
3527                 pause("equiesce", 1);
3528
3529         /* Finally, wait for the driver to reclaim all descriptors. */
3530         while (eq->cidx != eq->pidx)
3531                 pause("dquiesce", 1);
3532 }
3533
3534 static void
3535 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3536 {
3537
3538         /* XXXTX */
3539 }
3540
3541 static void
3542 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3543 {
3544         (void) sc;      /* unused */
3545
3546         /* Synchronize with the interrupt handler */
3547         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3548                 pause("iqfree", 1);
3549 }
3550
3551 static void
3552 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3553 {
3554         mtx_lock(&sc->sfl_lock);
3555         FL_LOCK(fl);
3556         fl->flags |= FL_DOOMED;
3557         FL_UNLOCK(fl);
3558         mtx_unlock(&sc->sfl_lock);
3559
3560         callout_drain(&sc->sfl_callout);
3561         KASSERT((fl->flags & FL_STARVING) == 0,
3562             ("%s: still starving", __func__));
3563 }
3564
3565 static int
3566 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3567     driver_intr_t *handler, void *arg, char *name)
3568 {
3569         int rc;
3570
3571         irq->rid = rid;
3572         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3573             RF_SHAREABLE | RF_ACTIVE);
3574         if (irq->res == NULL) {
3575                 device_printf(sc->dev,
3576                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3577                 return (ENOMEM);
3578         }
3579
3580         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3581             NULL, handler, arg, &irq->tag);
3582         if (rc != 0) {
3583                 device_printf(sc->dev,
3584                     "failed to setup interrupt for rid %d, name %s: %d\n",
3585                     rid, name, rc);
3586         } else if (name)
3587                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3588
3589         return (rc);
3590 }
3591
3592 static int
3593 t4_free_irq(struct adapter *sc, struct irq *irq)
3594 {
3595         if (irq->tag)
3596                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3597         if (irq->res)
3598                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3599
3600         bzero(irq, sizeof(*irq));
3601
3602         return (0);
3603 }
3604
3605 static void
3606 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3607     unsigned int end)
3608 {
3609         uint32_t *p = (uint32_t *)(buf + start);
3610
3611         for ( ; start <= end; start += sizeof(uint32_t))
3612                 *p++ = t4_read_reg(sc, start);
3613 }
3614
3615 static void
3616 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3617 {
3618         int i, n;
3619         const unsigned int *reg_ranges;
3620         static const unsigned int t4_reg_ranges[] = {
3621                 0x1008, 0x1108,
3622                 0x1180, 0x11b4,
3623                 0x11fc, 0x123c,
3624                 0x1300, 0x173c,
3625                 0x1800, 0x18fc,
3626                 0x3000, 0x30d8,
3627                 0x30e0, 0x5924,
3628                 0x5960, 0x59d4,
3629                 0x5a00, 0x5af8,
3630                 0x6000, 0x6098,
3631                 0x6100, 0x6150,
3632                 0x6200, 0x6208,
3633                 0x6240, 0x6248,
3634                 0x6280, 0x6338,
3635                 0x6370, 0x638c,
3636                 0x6400, 0x643c,
3637                 0x6500, 0x6524,
3638                 0x6a00, 0x6a38,
3639                 0x6a60, 0x6a78,
3640                 0x6b00, 0x6b84,
3641                 0x6bf0, 0x6c84,
3642                 0x6cf0, 0x6d84,
3643                 0x6df0, 0x6e84,
3644                 0x6ef0, 0x6f84,
3645                 0x6ff0, 0x7084,
3646                 0x70f0, 0x7184,
3647                 0x71f0, 0x7284,
3648                 0x72f0, 0x7384,
3649                 0x73f0, 0x7450,
3650                 0x7500, 0x7530,
3651                 0x7600, 0x761c,
3652                 0x7680, 0x76cc,
3653                 0x7700, 0x7798,
3654                 0x77c0, 0x77fc,
3655                 0x7900, 0x79fc,
3656                 0x7b00, 0x7c38,
3657                 0x7d00, 0x7efc,
3658                 0x8dc0, 0x8e1c,
3659                 0x8e30, 0x8e78,
3660                 0x8ea0, 0x8f6c,
3661                 0x8fc0, 0x9074,
3662                 0x90fc, 0x90fc,
3663                 0x9400, 0x9458,
3664                 0x9600, 0x96bc,
3665                 0x9800, 0x9808,
3666                 0x9820, 0x983c,
3667                 0x9850, 0x9864,
3668                 0x9c00, 0x9c6c,
3669                 0x9c80, 0x9cec,
3670                 0x9d00, 0x9d6c,
3671                 0x9d80, 0x9dec,
3672                 0x9e00, 0x9e6c,
3673                 0x9e80, 0x9eec,
3674                 0x9f00, 0x9f6c,
3675                 0x9f80, 0x9fec,
3676                 0xd004, 0xd03c,
3677                 0xdfc0, 0xdfe0,
3678                 0xe000, 0xea7c,
3679                 0xf000, 0x11110,
3680                 0x11118, 0x11190,
3681                 0x19040, 0x1906c,
3682                 0x19078, 0x19080,
3683                 0x1908c, 0x19124,
3684                 0x19150, 0x191b0,
3685                 0x191d0, 0x191e8,
3686                 0x19238, 0x1924c,
3687                 0x193f8, 0x19474,
3688                 0x19490, 0x194f8,
3689                 0x19800, 0x19f30,
3690                 0x1a000, 0x1a06c,
3691                 0x1a0b0, 0x1a120,
3692                 0x1a128, 0x1a138,
3693                 0x1a190, 0x1a1c4,
3694                 0x1a1fc, 0x1a1fc,
3695                 0x1e040, 0x1e04c,
3696                 0x1e284, 0x1e28c,
3697                 0x1e2c0, 0x1e2c0,
3698                 0x1e2e0, 0x1e2e0,
3699                 0x1e300, 0x1e384,
3700                 0x1e3c0, 0x1e3c8,
3701                 0x1e440, 0x1e44c,
3702                 0x1e684, 0x1e68c,
3703                 0x1e6c0, 0x1e6c0,
3704                 0x1e6e0, 0x1e6e0,
3705                 0x1e700, 0x1e784,
3706                 0x1e7c0, 0x1e7c8,
3707                 0x1e840, 0x1e84c,
3708                 0x1ea84, 0x1ea8c,
3709                 0x1eac0, 0x1eac0,
3710                 0x1eae0, 0x1eae0,
3711                 0x1eb00, 0x1eb84,
3712                 0x1ebc0, 0x1ebc8,
3713                 0x1ec40, 0x1ec4c,
3714                 0x1ee84, 0x1ee8c,
3715                 0x1eec0, 0x1eec0,
3716                 0x1eee0, 0x1eee0,
3717                 0x1ef00, 0x1ef84,
3718                 0x1efc0, 0x1efc8,
3719                 0x1f040, 0x1f04c,
3720                 0x1f284, 0x1f28c,
3721                 0x1f2c0, 0x1f2c0,
3722                 0x1f2e0, 0x1f2e0,
3723                 0x1f300, 0x1f384,
3724                 0x1f3c0, 0x1f3c8,
3725                 0x1f440, 0x1f44c,
3726                 0x1f684, 0x1f68c,
3727                 0x1f6c0, 0x1f6c0,
3728                 0x1f6e0, 0x1f6e0,
3729                 0x1f700, 0x1f784,
3730                 0x1f7c0, 0x1f7c8,
3731                 0x1f840, 0x1f84c,
3732                 0x1fa84, 0x1fa8c,
3733                 0x1fac0, 0x1fac0,
3734                 0x1fae0, 0x1fae0,
3735                 0x1fb00, 0x1fb84,
3736                 0x1fbc0, 0x1fbc8,
3737                 0x1fc40, 0x1fc4c,
3738                 0x1fe84, 0x1fe8c,
3739                 0x1fec0, 0x1fec0,
3740                 0x1fee0, 0x1fee0,
3741                 0x1ff00, 0x1ff84,
3742                 0x1ffc0, 0x1ffc8,
3743                 0x20000, 0x2002c,
3744                 0x20100, 0x2013c,
3745                 0x20190, 0x201c8,
3746                 0x20200, 0x20318,
3747                 0x20400, 0x20528,
3748                 0x20540, 0x20614,
3749                 0x21000, 0x21040,
3750                 0x2104c, 0x21060,
3751                 0x210c0, 0x210ec,
3752                 0x21200, 0x21268,
3753                 0x21270, 0x21284,
3754                 0x212fc, 0x21388,
3755                 0x21400, 0x21404,
3756                 0x21500, 0x21518,
3757                 0x2152c, 0x2153c,
3758                 0x21550, 0x21554,
3759                 0x21600, 0x21600,
3760                 0x21608, 0x21628,
3761                 0x21630, 0x2163c,
3762                 0x21700, 0x2171c,
3763                 0x21780, 0x2178c,
3764                 0x21800, 0x21c38,
3765                 0x21c80, 0x21d7c,
3766                 0x21e00, 0x21e04,
3767                 0x22000, 0x2202c,
3768                 0x22100, 0x2213c,
3769                 0x22190, 0x221c8,
3770                 0x22200, 0x22318,
3771                 0x22400, 0x22528,
3772                 0x22540, 0x22614,
3773                 0x23000, 0x23040,
3774                 0x2304c, 0x23060,
3775                 0x230c0, 0x230ec,
3776                 0x23200, 0x23268,
3777                 0x23270, 0x23284,
3778                 0x232fc, 0x23388,
3779                 0x23400, 0x23404,
3780                 0x23500, 0x23518,
3781                 0x2352c, 0x2353c,
3782                 0x23550, 0x23554,
3783                 0x23600, 0x23600,
3784                 0x23608, 0x23628,
3785                 0x23630, 0x2363c,
3786                 0x23700, 0x2371c,
3787                 0x23780, 0x2378c,
3788                 0x23800, 0x23c38,
3789                 0x23c80, 0x23d7c,
3790                 0x23e00, 0x23e04,
3791                 0x24000, 0x2402c,
3792                 0x24100, 0x2413c,
3793                 0x24190, 0x241c8,
3794                 0x24200, 0x24318,
3795                 0x24400, 0x24528,
3796                 0x24540, 0x24614,
3797                 0x25000, 0x25040,
3798                 0x2504c, 0x25060,
3799                 0x250c0, 0x250ec,
3800                 0x25200, 0x25268,
3801                 0x25270, 0x25284,
3802                 0x252fc, 0x25388,
3803                 0x25400, 0x25404,
3804                 0x25500, 0x25518,
3805                 0x2552c, 0x2553c,
3806                 0x25550, 0x25554,
3807                 0x25600, 0x25600,
3808                 0x25608, 0x25628,
3809                 0x25630, 0x2563c,
3810                 0x25700, 0x2571c,
3811                 0x25780, 0x2578c,
3812                 0x25800, 0x25c38,
3813                 0x25c80, 0x25d7c,
3814                 0x25e00, 0x25e04,
3815                 0x26000, 0x2602c,
3816                 0x26100, 0x2613c,
3817                 0x26190, 0x261c8,
3818                 0x26200, 0x26318,
3819                 0x26400, 0x26528,
3820                 0x26540, 0x26614,
3821                 0x27000, 0x27040,
3822                 0x2704c, 0x27060,
3823                 0x270c0, 0x270ec,
3824                 0x27200, 0x27268,
3825                 0x27270, 0x27284,
3826                 0x272fc, 0x27388,
3827                 0x27400, 0x27404,
3828                 0x27500, 0x27518,
3829                 0x2752c, 0x2753c,
3830                 0x27550, 0x27554,
3831                 0x27600, 0x27600,
3832                 0x27608, 0x27628,
3833                 0x27630, 0x2763c,
3834                 0x27700, 0x2771c,
3835                 0x27780, 0x2778c,
3836                 0x27800, 0x27c38,
3837                 0x27c80, 0x27d7c,
3838                 0x27e00, 0x27e04
3839         };
3840         static const unsigned int t5_reg_ranges[] = {
3841                 0x1008, 0x1148,
3842                 0x1180, 0x11b4,
3843                 0x11fc, 0x123c,
3844                 0x1280, 0x173c,
3845                 0x1800, 0x18fc,
3846                 0x3000, 0x3028,
3847                 0x3060, 0x30d8,
3848                 0x30e0, 0x30fc,
3849                 0x3140, 0x357c,
3850                 0x35a8, 0x35cc,
3851                 0x35ec, 0x35ec,
3852                 0x3600, 0x5624,
3853                 0x56cc, 0x575c,
3854                 0x580c, 0x5814,
3855                 0x5890, 0x58bc,
3856                 0x5940, 0x59dc,
3857                 0x59fc, 0x5a18,
3858                 0x5a60, 0x5a9c,
3859                 0x5b94, 0x5bfc,
3860                 0x6000, 0x6040,
3861                 0x6058, 0x614c,
3862                 0x7700, 0x7798,
3863                 0x77c0, 0x78fc,
3864                 0x7b00, 0x7c54,
3865                 0x7d00, 0x7efc,
3866                 0x8dc0, 0x8de0,
3867                 0x8df8, 0x8e84,
3868                 0x8ea0, 0x8f84,
3869                 0x8fc0, 0x90f8,
3870                 0x9400, 0x9470,
3871                 0x9600, 0x96f4,
3872                 0x9800, 0x9808,
3873                 0x9820, 0x983c,
3874                 0x9850, 0x9864,
3875                 0x9c00, 0x9c6c,
3876                 0x9c80, 0x9cec,
3877                 0x9d00, 0x9d6c,
3878                 0x9d80, 0x9dec,
3879                 0x9e00, 0x9e6c,
3880                 0x9e80, 0x9eec,
3881                 0x9f00, 0x9f6c,
3882                 0x9f80, 0xa020,
3883                 0xd004, 0xd03c,
3884                 0xdfc0, 0xdfe0,
3885                 0xe000, 0x11088,
3886                 0x1109c, 0x11110,
3887                 0x11118, 0x1117c,
3888                 0x11190, 0x11204,
3889                 0x19040, 0x1906c,
3890                 0x19078, 0x19080,
3891                 0x1908c, 0x19124,
3892                 0x19150, 0x191b0,
3893                 0x191d0, 0x191e8,
3894                 0x19238, 0x19290,
3895                 0x193f8, 0x19474,
3896                 0x19490, 0x194cc,
3897                 0x194f0, 0x194f8,
3898                 0x19c00, 0x19c60,
3899                 0x19c94, 0x19e10,
3900                 0x19e50, 0x19f34,
3901                 0x19f40, 0x19f50,
3902                 0x19f90, 0x19fe4,
3903                 0x1a000, 0x1a06c,
3904                 0x1a0b0, 0x1a120,
3905                 0x1a128, 0x1a138,
3906                 0x1a190, 0x1a1c4,
3907                 0x1a1fc, 0x1a1fc,
3908                 0x1e008, 0x1e00c,
3909                 0x1e040, 0x1e04c,
3910                 0x1e284, 0x1e290,
3911                 0x1e2c0, 0x1e2c0,
3912                 0x1e2e0, 0x1e2e0,
3913                 0x1e300, 0x1e384,
3914                 0x1e3c0, 0x1e3c8,
3915                 0x1e408, 0x1e40c,
3916                 0x1e440, 0x1e44c,
3917                 0x1e684, 0x1e690,
3918                 0x1e6c0, 0x1e6c0,
3919                 0x1e6e0, 0x1e6e0,
3920                 0x1e700, 0x1e784,
3921                 0x1e7c0, 0x1e7c8,
3922                 0x1e808, 0x1e80c,
3923                 0x1e840, 0x1e84c,
3924                 0x1ea84, 0x1ea90,
3925                 0x1eac0, 0x1eac0,
3926                 0x1eae0, 0x1eae0,
3927                 0x1eb00, 0x1eb84,
3928                 0x1ebc0, 0x1ebc8,
3929                 0x1ec08, 0x1ec0c,
3930                 0x1ec40, 0x1ec4c,
3931                 0x1ee84, 0x1ee90,
3932                 0x1eec0, 0x1eec0,
3933                 0x1eee0, 0x1eee0,
3934                 0x1ef00, 0x1ef84,
3935                 0x1efc0, 0x1efc8,
3936                 0x1f008, 0x1f00c,
3937                 0x1f040, 0x1f04c,
3938                 0x1f284, 0x1f290,
3939                 0x1f2c0, 0x1f2c0,
3940                 0x1f2e0, 0x1f2e0,
3941                 0x1f300, 0x1f384,
3942                 0x1f3c0, 0x1f3c8,
3943                 0x1f408, 0x1f40c,
3944                 0x1f440, 0x1f44c,
3945                 0x1f684, 0x1f690,
3946                 0x1f6c0, 0x1f6c0,
3947                 0x1f6e0, 0x1f6e0,
3948                 0x1f700, 0x1f784,
3949                 0x1f7c0, 0x1f7c8,
3950                 0x1f808, 0x1f80c,
3951                 0x1f840, 0x1f84c,
3952                 0x1fa84, 0x1fa90,
3953                 0x1fac0, 0x1fac0,
3954                 0x1fae0, 0x1fae0,
3955                 0x1fb00, 0x1fb84,
3956                 0x1fbc0, 0x1fbc8,
3957                 0x1fc08, 0x1fc0c,
3958                 0x1fc40, 0x1fc4c,
3959                 0x1fe84, 0x1fe90,
3960                 0x1fec0, 0x1fec0,
3961                 0x1fee0, 0x1fee0,
3962                 0x1ff00, 0x1ff84,
3963                 0x1ffc0, 0x1ffc8,
3964                 0x30000, 0x30030,
3965                 0x30100, 0x30144,
3966                 0x30190, 0x301d0,
3967                 0x30200, 0x30318,
3968                 0x30400, 0x3052c,
3969                 0x30540, 0x3061c,
3970                 0x30800, 0x30834,
3971                 0x308c0, 0x30908,
3972                 0x30910, 0x309ac,
3973                 0x30a00, 0x30a2c,
3974                 0x30a44, 0x30a50,
3975                 0x30a74, 0x30c24,
3976                 0x30d00, 0x30d00,
3977                 0x30d08, 0x30d14,
3978                 0x30d1c, 0x30d20,
3979                 0x30d3c, 0x30d50,
3980                 0x31200, 0x3120c,
3981                 0x31220, 0x31220,
3982                 0x31240, 0x31240,
3983                 0x31600, 0x3160c,
3984                 0x31a00, 0x31a1c,
3985                 0x31e00, 0x31e20,
3986                 0x31e38, 0x31e3c,
3987                 0x31e80, 0x31e80,
3988                 0x31e88, 0x31ea8,
3989                 0x31eb0, 0x31eb4,
3990                 0x31ec8, 0x31ed4,
3991                 0x31fb8, 0x32004,
3992                 0x32200, 0x32200,
3993                 0x32208, 0x32240,
3994                 0x32248, 0x32280,
3995                 0x32288, 0x322c0,
3996                 0x322c8, 0x322fc,
3997                 0x32600, 0x32630,
3998                 0x32a00, 0x32abc,
3999                 0x32b00, 0x32b70,
4000                 0x33000, 0x33048,
4001                 0x33060, 0x3309c,
4002                 0x330f0, 0x33148,
4003                 0x33160, 0x3319c,
4004                 0x331f0, 0x332e4,
4005                 0x332f8, 0x333e4,
4006                 0x333f8, 0x33448,
4007                 0x33460, 0x3349c,
4008                 0x334f0, 0x33548,
4009                 0x33560, 0x3359c,
4010                 0x335f0, 0x336e4,
4011                 0x336f8, 0x337e4,
4012                 0x337f8, 0x337fc,
4013                 0x33814, 0x33814,
4014                 0x3382c, 0x3382c,
4015                 0x33880, 0x3388c,
4016                 0x338e8, 0x338ec,
4017                 0x33900, 0x33948,
4018                 0x33960, 0x3399c,
4019                 0x339f0, 0x33ae4,
4020                 0x33af8, 0x33b10,
4021                 0x33b28, 0x33b28,
4022                 0x33b3c, 0x33b50,
4023                 0x33bf0, 0x33c10,
4024                 0x33c28, 0x33c28,
4025                 0x33c3c, 0x33c50,
4026                 0x33cf0, 0x33cfc,
4027                 0x34000, 0x34030,
4028                 0x34100, 0x34144,
4029                 0x34190, 0x341d0,
4030                 0x34200, 0x34318,
4031                 0x34400, 0x3452c,
4032                 0x34540, 0x3461c,
4033                 0x34800, 0x34834,
4034                 0x348c0, 0x34908,
4035                 0x34910, 0x349ac,
4036                 0x34a00, 0x34a2c,
4037                 0x34a44, 0x34a50,
4038                 0x34a74, 0x34c24,
4039                 0x34d00, 0x34d00,
4040                 0x34d08, 0x34d14,
4041                 0x34d1c, 0x34d20,
4042                 0x34d3c, 0x34d50,
4043                 0x35200, 0x3520c,
4044                 0x35220, 0x35220,
4045                 0x35240, 0x35240,
4046                 0x35600, 0x3560c,
4047                 0x35a00, 0x35a1c,
4048                 0x35e00, 0x35e20,
4049                 0x35e38, 0x35e3c,
4050                 0x35e80, 0x35e80,
4051                 0x35e88, 0x35ea8,
4052                 0x35eb0, 0x35eb4,
4053                 0x35ec8, 0x35ed4,
4054                 0x35fb8, 0x36004,
4055                 0x36200, 0x36200,
4056                 0x36208, 0x36240,
4057                 0x36248, 0x36280,
4058                 0x36288, 0x362c0,
4059                 0x362c8, 0x362fc,
4060                 0x36600, 0x36630,
4061                 0x36a00, 0x36abc,
4062                 0x36b00, 0x36b70,
4063                 0x37000, 0x37048,
4064                 0x37060, 0x3709c,
4065                 0x370f0, 0x37148,
4066                 0x37160, 0x3719c,
4067                 0x371f0, 0x372e4,
4068                 0x372f8, 0x373e4,
4069                 0x373f8, 0x37448,
4070                 0x37460, 0x3749c,
4071                 0x374f0, 0x37548,
4072                 0x37560, 0x3759c,
4073                 0x375f0, 0x376e4,
4074                 0x376f8, 0x377e4,
4075                 0x377f8, 0x377fc,
4076                 0x37814, 0x37814,
4077                 0x3782c, 0x3782c,
4078                 0x37880, 0x3788c,
4079                 0x378e8, 0x378ec,
4080                 0x37900, 0x37948,
4081                 0x37960, 0x3799c,
4082                 0x379f0, 0x37ae4,
4083                 0x37af8, 0x37b10,
4084                 0x37b28, 0x37b28,
4085                 0x37b3c, 0x37b50,
4086                 0x37bf0, 0x37c10,
4087                 0x37c28, 0x37c28,
4088                 0x37c3c, 0x37c50,
4089                 0x37cf0, 0x37cfc,
4090                 0x38000, 0x38030,
4091                 0x38100, 0x38144,
4092                 0x38190, 0x381d0,
4093                 0x38200, 0x38318,
4094                 0x38400, 0x3852c,
4095                 0x38540, 0x3861c,
4096                 0x38800, 0x38834,
4097                 0x388c0, 0x38908,
4098                 0x38910, 0x389ac,
4099                 0x38a00, 0x38a2c,
4100                 0x38a44, 0x38a50,
4101                 0x38a74, 0x38c24,
4102                 0x38d00, 0x38d00,
4103                 0x38d08, 0x38d14,
4104                 0x38d1c, 0x38d20,
4105                 0x38d3c, 0x38d50,
4106                 0x39200, 0x3920c,
4107                 0x39220, 0x39220,
4108                 0x39240, 0x39240,
4109                 0x39600, 0x3960c,
4110                 0x39a00, 0x39a1c,
4111                 0x39e00, 0x39e20,
4112                 0x39e38, 0x39e3c,
4113                 0x39e80, 0x39e80,
4114                 0x39e88, 0x39ea8,
4115                 0x39eb0, 0x39eb4,
4116                 0x39ec8, 0x39ed4,
4117                 0x39fb8, 0x3a004,
4118                 0x3a200, 0x3a200,
4119                 0x3a208, 0x3a240,
4120                 0x3a248, 0x3a280,
4121                 0x3a288, 0x3a2c0,
4122                 0x3a2c8, 0x3a2fc,
4123                 0x3a600, 0x3a630,
4124                 0x3aa00, 0x3aabc,
4125                 0x3ab00, 0x3ab70,
4126                 0x3b000, 0x3b048,
4127                 0x3b060, 0x3b09c,
4128                 0x3b0f0, 0x3b148,
4129                 0x3b160, 0x3b19c,
4130                 0x3b1f0, 0x3b2e4,
4131                 0x3b2f8, 0x3b3e4,
4132                 0x3b3f8, 0x3b448,
4133                 0x3b460, 0x3b49c,
4134                 0x3b4f0, 0x3b548,
4135                 0x3b560, 0x3b59c,
4136                 0x3b5f0, 0x3b6e4,
4137                 0x3b6f8, 0x3b7e4,
4138                 0x3b7f8, 0x3b7fc,
4139                 0x3b814, 0x3b814,
4140                 0x3b82c, 0x3b82c,
4141                 0x3b880, 0x3b88c,
4142                 0x3b8e8, 0x3b8ec,
4143                 0x3b900, 0x3b948,
4144                 0x3b960, 0x3b99c,
4145                 0x3b9f0, 0x3bae4,
4146                 0x3baf8, 0x3bb10,
4147                 0x3bb28, 0x3bb28,
4148                 0x3bb3c, 0x3bb50,
4149                 0x3bbf0, 0x3bc10,
4150                 0x3bc28, 0x3bc28,
4151                 0x3bc3c, 0x3bc50,
4152                 0x3bcf0, 0x3bcfc,
4153                 0x3c000, 0x3c030,
4154                 0x3c100, 0x3c144,
4155                 0x3c190, 0x3c1d0,
4156                 0x3c200, 0x3c318,
4157                 0x3c400, 0x3c52c,
4158                 0x3c540, 0x3c61c,
4159                 0x3c800, 0x3c834,
4160                 0x3c8c0, 0x3c908,
4161                 0x3c910, 0x3c9ac,
4162                 0x3ca00, 0x3ca2c,
4163                 0x3ca44, 0x3ca50,
4164                 0x3ca74, 0x3cc24,
4165                 0x3cd00, 0x3cd00,
4166                 0x3cd08, 0x3cd14,
4167                 0x3cd1c, 0x3cd20,
4168                 0x3cd3c, 0x3cd50,
4169                 0x3d200, 0x3d20c,
4170                 0x3d220, 0x3d220,
4171                 0x3d240, 0x3d240,
4172                 0x3d600, 0x3d60c,
4173                 0x3da00, 0x3da1c,
4174                 0x3de00, 0x3de20,
4175                 0x3de38, 0x3de3c,
4176                 0x3de80, 0x3de80,
4177                 0x3de88, 0x3dea8,
4178                 0x3deb0, 0x3deb4,
4179                 0x3dec8, 0x3ded4,
4180                 0x3dfb8, 0x3e004,
4181                 0x3e200, 0x3e200,
4182                 0x3e208, 0x3e240,
4183                 0x3e248, 0x3e280,
4184                 0x3e288, 0x3e2c0,
4185                 0x3e2c8, 0x3e2fc,
4186                 0x3e600, 0x3e630,
4187                 0x3ea00, 0x3eabc,
4188                 0x3eb00, 0x3eb70,
4189                 0x3f000, 0x3f048,
4190                 0x3f060, 0x3f09c,
4191                 0x3f0f0, 0x3f148,
4192                 0x3f160, 0x3f19c,
4193                 0x3f1f0, 0x3f2e4,
4194                 0x3f2f8, 0x3f3e4,
4195                 0x3f3f8, 0x3f448,
4196                 0x3f460, 0x3f49c,
4197                 0x3f4f0, 0x3f548,
4198                 0x3f560, 0x3f59c,
4199                 0x3f5f0, 0x3f6e4,
4200                 0x3f6f8, 0x3f7e4,
4201                 0x3f7f8, 0x3f7fc,
4202                 0x3f814, 0x3f814,
4203                 0x3f82c, 0x3f82c,
4204                 0x3f880, 0x3f88c,
4205                 0x3f8e8, 0x3f8ec,
4206                 0x3f900, 0x3f948,
4207                 0x3f960, 0x3f99c,
4208                 0x3f9f0, 0x3fae4,
4209                 0x3faf8, 0x3fb10,
4210                 0x3fb28, 0x3fb28,
4211                 0x3fb3c, 0x3fb50,
4212                 0x3fbf0, 0x3fc10,
4213                 0x3fc28, 0x3fc28,
4214                 0x3fc3c, 0x3fc50,
4215                 0x3fcf0, 0x3fcfc,
4216                 0x40000, 0x4000c,
4217                 0x40040, 0x40068,
4218                 0x4007c, 0x40144,
4219                 0x40180, 0x4018c,
4220                 0x40200, 0x40298,
4221                 0x402ac, 0x4033c,
4222                 0x403f8, 0x403fc,
4223                 0x41304, 0x413c4,
4224                 0x41400, 0x4141c,
4225                 0x41480, 0x414d0,
4226                 0x44000, 0x44078,
4227                 0x440c0, 0x44278,
4228                 0x442c0, 0x44478,
4229                 0x444c0, 0x44678,
4230                 0x446c0, 0x44878,
4231                 0x448c0, 0x449fc,
4232                 0x45000, 0x45068,
4233                 0x45080, 0x45084,
4234                 0x450a0, 0x450b0,
4235                 0x45200, 0x45268,
4236                 0x45280, 0x45284,
4237                 0x452a0, 0x452b0,
4238                 0x460c0, 0x460e4,
4239                 0x47000, 0x4708c,
4240                 0x47200, 0x47250,
4241                 0x47400, 0x47420,
4242                 0x47600, 0x47618,
4243                 0x47800, 0x47814,
4244                 0x48000, 0x4800c,
4245                 0x48040, 0x48068,
4246                 0x4807c, 0x48144,
4247                 0x48180, 0x4818c,
4248                 0x48200, 0x48298,
4249                 0x482ac, 0x4833c,
4250                 0x483f8, 0x483fc,
4251                 0x49304, 0x493c4,
4252                 0x49400, 0x4941c,
4253                 0x49480, 0x494d0,
4254                 0x4c000, 0x4c078,
4255                 0x4c0c0, 0x4c278,
4256                 0x4c2c0, 0x4c478,
4257                 0x4c4c0, 0x4c678,
4258                 0x4c6c0, 0x4c878,
4259                 0x4c8c0, 0x4c9fc,
4260                 0x4d000, 0x4d068,
4261                 0x4d080, 0x4d084,
4262                 0x4d0a0, 0x4d0b0,
4263                 0x4d200, 0x4d268,
4264                 0x4d280, 0x4d284,
4265                 0x4d2a0, 0x4d2b0,
4266                 0x4e0c0, 0x4e0e4,
4267                 0x4f000, 0x4f08c,
4268                 0x4f200, 0x4f250,
4269                 0x4f400, 0x4f420,
4270                 0x4f600, 0x4f618,
4271                 0x4f800, 0x4f814,
4272                 0x50000, 0x500cc,
4273                 0x50400, 0x50400,
4274                 0x50800, 0x508cc,
4275                 0x50c00, 0x50c00,
4276                 0x51000, 0x5101c,
4277                 0x51300, 0x51308,
4278         };
4279
4280         if (is_t4(sc)) {
4281                 reg_ranges = &t4_reg_ranges[0];
4282                 n = nitems(t4_reg_ranges);
4283         } else {
4284                 reg_ranges = &t5_reg_ranges[0];
4285                 n = nitems(t5_reg_ranges);
4286         }
4287
4288         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4289         for (i = 0; i < n; i += 2)
4290                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4291 }
4292
4293 static void
4294 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4295 {
4296         struct ifnet *ifp = pi->ifp;
4297         struct sge_txq *txq;
4298         int i, drops;
4299         struct port_stats *s = &pi->stats;
4300         struct timeval tv;
4301         const struct timeval interval = {0, 250000};    /* 250ms */
4302
4303         getmicrotime(&tv);
4304         timevalsub(&tv, &interval);
4305         if (timevalcmp(&tv, &pi->last_refreshed, <))
4306                 return;
4307
4308         t4_get_port_stats(sc, pi->tx_chan, s);
4309
4310         ifp->if_opackets = s->tx_frames - s->tx_pause;
4311         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4312         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4313         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4314         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4315         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4316         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4317             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4318             s->rx_trunc3;
4319         for (i = 0; i < NCHAN; i++) {
4320                 if (pi->rx_chan_map & (1 << i)) {
4321                         uint32_t v;
4322
4323                         mtx_lock(&sc->regwin_lock);
4324                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4325                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4326                         mtx_unlock(&sc->regwin_lock);
4327                         ifp->if_iqdrops += v;
4328                 }
4329         }
4330
4331         drops = s->tx_drop;
4332         for_each_txq(pi, i, txq)
4333                 drops += counter_u64_fetch(txq->r->drops);
4334         ifp->if_snd.ifq_drops = drops;
4335
4336         ifp->if_oerrors = s->tx_error_frames;
4337         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4338             s->rx_fcs_err + s->rx_len_err;
4339
4340         getmicrotime(&pi->last_refreshed);
4341 }
4342
4343 static void
4344 cxgbe_tick(void *arg)
4345 {
4346         struct port_info *pi = arg;
4347         struct adapter *sc = pi->adapter;
4348         struct ifnet *ifp = pi->ifp;
4349
4350         PORT_LOCK(pi);
4351         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4352                 PORT_UNLOCK(pi);
4353                 return; /* without scheduling another callout */
4354         }
4355
4356         cxgbe_refresh_stats(sc, pi);
4357
4358         callout_schedule(&pi->tick, hz);
4359         PORT_UNLOCK(pi);
4360 }
4361
4362 static void
4363 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4364 {
4365         struct ifnet *vlan;
4366
4367         if (arg != ifp || ifp->if_type != IFT_ETHER)
4368                 return;
4369
4370         vlan = VLAN_DEVAT(ifp, vid);
4371         VLAN_SETCOOKIE(vlan, ifp);
4372 }
4373
4374 static int
4375 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4376 {
4377
4378 #ifdef INVARIANTS
4379         panic("%s: opcode 0x%02x on iq %p with payload %p",
4380             __func__, rss->opcode, iq, m);
4381 #else
4382         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4383             __func__, rss->opcode, iq, m);
4384         m_freem(m);
4385 #endif
4386         return (EDOOFUS);
4387 }
4388
4389 int
4390 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4391 {
4392         uintptr_t *loc, new;
4393
4394         if (opcode >= nitems(sc->cpl_handler))
4395                 return (EINVAL);
4396
4397         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4398         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4399         atomic_store_rel_ptr(loc, new);
4400
4401         return (0);
4402 }
4403
4404 static int
4405 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4406 {
4407
4408 #ifdef INVARIANTS
4409         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4410 #else
4411         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4412             __func__, iq, ctrl);
4413 #endif
4414         return (EDOOFUS);
4415 }
4416
4417 int
4418 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4419 {
4420         uintptr_t *loc, new;
4421
4422         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4423         loc = (uintptr_t *) &sc->an_handler;
4424         atomic_store_rel_ptr(loc, new);
4425
4426         return (0);
4427 }
4428
4429 static int
4430 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4431 {
4432         const struct cpl_fw6_msg *cpl =
4433             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4434
4435 #ifdef INVARIANTS
4436         panic("%s: fw_msg type %d", __func__, cpl->type);
4437 #else
4438         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4439 #endif
4440         return (EDOOFUS);
4441 }
4442
4443 int
4444 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4445 {
4446         uintptr_t *loc, new;
4447
4448         if (type >= nitems(sc->fw_msg_handler))
4449                 return (EINVAL);
4450
4451         /*
4452          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4453          * handler dispatch table.  Reject any attempt to install a handler for
4454          * this subtype.
4455          */
4456         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4457                 return (EINVAL);
4458
4459         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4460         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4461         atomic_store_rel_ptr(loc, new);
4462
4463         return (0);
4464 }
4465
4466 static int
4467 t4_sysctls(struct adapter *sc)
4468 {
4469         struct sysctl_ctx_list *ctx;
4470         struct sysctl_oid *oid;
4471         struct sysctl_oid_list *children, *c0;
4472         static char *caps[] = {
4473                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4474                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4475                     "\6HASHFILTER\7ETHOFLD",
4476                 "\20\1TOE",                             /* caps[2] toecaps */
4477                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4478                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4479                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4480                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4481                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4482                     "\4PO_INITIAOR\5PO_TARGET"
4483         };
4484         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4485
4486         ctx = device_get_sysctl_ctx(sc->dev);
4487
4488         /*
4489          * dev.t4nex.X.
4490          */
4491         oid = device_get_sysctl_tree(sc->dev);
4492         c0 = children = SYSCTL_CHILDREN(oid);
4493
4494         sc->sc_do_rxcopy = 1;
4495         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4496             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4497
4498         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4499             sc->params.nports, "# of ports");
4500
4501         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4502             NULL, chip_rev(sc), "chip hardware revision");
4503
4504         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4505             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4506
4507         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4508             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4509
4510         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4511             sc->cfcsum, "config file checksum");
4512
4513         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4514             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4515             sysctl_bitfield, "A", "available doorbells");
4516
4517         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4518             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4519             sysctl_bitfield, "A", "available link capabilities");
4520
4521         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4522             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4523             sysctl_bitfield, "A", "available NIC capabilities");
4524
4525         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4526             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4527             sysctl_bitfield, "A", "available TCP offload capabilities");
4528
4529         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4530             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4531             sysctl_bitfield, "A", "available RDMA capabilities");
4532
4533         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4534             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4535             sysctl_bitfield, "A", "available iSCSI capabilities");
4536
4537         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4538             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4539             sysctl_bitfield, "A", "available FCoE capabilities");
4540
4541         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4542             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4543
4544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4545             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4546             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4547             "interrupt holdoff timer values (us)");
4548
4549         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4550             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4551             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4552             "interrupt holdoff packet counter values");
4553
4554         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4555             NULL, sc->tids.nftids, "number of filters");
4556
4557         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4558             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4559             "chip temperature (in Celsius)");
4560
4561         t4_sge_sysctls(sc, ctx, children);
4562
4563         sc->lro_timeout = 100;
4564         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4565             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4566
4567         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4568             &sc->debug_flags, 0, "flags to enable runtime debugging");
4569
4570 #ifdef SBUF_DRAIN
4571         /*
4572          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4573          */
4574         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4575             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4576             "logs and miscellaneous information");
4577         children = SYSCTL_CHILDREN(oid);
4578
4579         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4580             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4581             sysctl_cctrl, "A", "congestion control");
4582
4583         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4584             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4585             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4586
4587         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4588             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4589             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4592             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4593             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4594
4595         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4596             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4597             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4598
4599         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4600             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4601             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4602
4603         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4604             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4605             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4606
4607         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4608             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4609             sysctl_cim_la, "A", "CIM logic analyzer");
4610
4611         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4612             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4613             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4614
4615         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4616             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4617             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4618
4619         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4620             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4621             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4622
4623         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4624             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4625             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4626
4627         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4628             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4629             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4633             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4637             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4638
4639         if (is_t5(sc)) {
4640                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4641                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4642                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4643
4644                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4645                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4646                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4647         }
4648
4649         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4650             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4651             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4652
4653         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4654             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4655             sysctl_cim_qcfg, "A", "CIM queue configuration");
4656
4657         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4658             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4659             sysctl_cpl_stats, "A", "CPL statistics");
4660
4661         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4662             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4663             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4664
4665         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4666             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4667             sysctl_devlog, "A", "firmware's device log");
4668
4669         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4670             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4671             sysctl_fcoe_stats, "A", "FCoE statistics");
4672
4673         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4674             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4675             sysctl_hw_sched, "A", "hardware scheduler ");
4676
4677         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4678             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4679             sysctl_l2t, "A", "hardware L2 table");
4680
4681         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4682             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4683             sysctl_lb_stats, "A", "loopback statistics");
4684
4685         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4686             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4687             sysctl_meminfo, "A", "memory regions");
4688
4689         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4690             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4691             sysctl_mps_tcam, "A", "MPS TCAM entries");
4692
4693         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4694             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4695             sysctl_path_mtus, "A", "path MTUs");
4696
4697         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4698             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4699             sysctl_pm_stats, "A", "PM statistics");
4700
4701         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4702             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703             sysctl_rdma_stats, "A", "RDMA statistics");
4704
4705         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4706             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4707             sysctl_tcp_stats, "A", "TCP statistics");
4708
4709         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4710             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4711             sysctl_tids, "A", "TID information");
4712
4713         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4714             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4715             sysctl_tp_err_stats, "A", "TP error statistics");
4716
4717         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4718             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4719             sysctl_tp_la, "A", "TP logic analyzer");
4720
4721         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4722             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4723             sysctl_tx_rate, "A", "Tx rate");
4724
4725         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4726             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4727             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4728
4729         if (is_t5(sc)) {
4730                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4731                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4732                     sysctl_wcwr_stats, "A", "write combined work requests");
4733         }
4734 #endif
4735
4736 #ifdef TCP_OFFLOAD
4737         if (is_offload(sc)) {
4738                 /*
4739                  * dev.t4nex.X.toe.
4740                  */
4741                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4742                     NULL, "TOE parameters");
4743                 children = SYSCTL_CHILDREN(oid);
4744
4745                 sc->tt.sndbuf = 256 * 1024;
4746                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4747                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4748
4749                 sc->tt.ddp = 0;
4750                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4751                     &sc->tt.ddp, 0, "DDP allowed");
4752
4753                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4754                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4755                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4756
4757                 sc->tt.ddp_thres =
4758                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4759                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4760                     &sc->tt.ddp_thres, 0, "DDP threshold");
4761
4762                 sc->tt.rx_coalesce = 1;
4763                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4764                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4765
4766                 sc->tt.tx_align = 1;
4767                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4768                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4769         }
4770 #endif
4771
4772
4773         return (0);
4774 }
4775
4776 static int
4777 cxgbe_sysctls(struct port_info *pi)
4778 {
4779         struct sysctl_ctx_list *ctx;
4780         struct sysctl_oid *oid;
4781         struct sysctl_oid_list *children;
4782         struct adapter *sc = pi->adapter;
4783
4784         ctx = device_get_sysctl_ctx(pi->dev);
4785
4786         /*
4787          * dev.cxgbe.X.
4788          */
4789         oid = device_get_sysctl_tree(pi->dev);
4790         children = SYSCTL_CHILDREN(oid);
4791
4792         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4793            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4794         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4795                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4796                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4797                     "PHY temperature (in Celsius)");
4798                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4799                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4800                     "PHY firmware version");
4801         }
4802         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4803             &pi->nrxq, 0, "# of rx queues");
4804         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4805             &pi->ntxq, 0, "# of tx queues");
4806         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4807             &pi->first_rxq, 0, "index of first rx queue");
4808         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4809             &pi->first_txq, 0, "index of first tx queue");
4810         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4811             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4812             "Reserve queue 0 for non-flowid packets");
4813
4814 #ifdef TCP_OFFLOAD
4815         if (is_offload(sc)) {
4816                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4817                     &pi->nofldrxq, 0,
4818                     "# of rx queues for offloaded TCP connections");
4819                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4820                     &pi->nofldtxq, 0,
4821                     "# of tx queues for offloaded TCP connections");
4822                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4823                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4824                     "index of first TOE rx queue");
4825                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4826                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4827                     "index of first TOE tx queue");
4828         }
4829 #endif
4830 #ifdef DEV_NETMAP
4831         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4832             &pi->nnmrxq, 0, "# of rx queues for netmap");
4833         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4834             &pi->nnmtxq, 0, "# of tx queues for netmap");
4835         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4836             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4837             "index of first netmap rx queue");
4838         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4839             CTLFLAG_RD, &pi->first_nm_txq, 0,
4840             "index of first netmap tx queue");
4841 #endif
4842
4843         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4844             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4845             "holdoff timer index");
4846         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4847             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4848             "holdoff packet counter index");
4849
4850         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4851             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4852             "rx queue size");
4853         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4854             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4855             "tx queue size");
4856
4857         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4858             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4859             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4860
4861         /*
4862          * dev.cxgbe.X.stats.
4863          */
4864         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4865             NULL, "port statistics");
4866         children = SYSCTL_CHILDREN(oid);
4867         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4868             &pi->tx_parse_error, 0,
4869             "# of tx packets with invalid length or # of segments");
4870
4871 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4872         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4873             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4874             sysctl_handle_t4_reg64, "QU", desc)
4875
4876         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4878         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4880         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4882         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4884         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4886         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4887             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4888         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4889             "# of tx frames in this range",
4890             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4891         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4892             "# of tx frames in this range",
4893             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4894         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4895             "# of tx frames in this range",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4897         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4898             "# of tx frames in this range",
4899             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4900         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4901             "# of tx frames in this range",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4903         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4904             "# of tx frames in this range",
4905             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4906         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4907             "# of tx frames in this range",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4909         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4911         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4913         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4927         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4928             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4929
4930         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4932         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4934         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4936         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4938         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4940         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4942         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4944         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4945             "# of frames received with bad FCS",
4946             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4947         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4948             "# of frames received with length error",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4950         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4952         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4954         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4955             "# of rx frames in this range",
4956             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4957         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4958             "# of rx frames in this range",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4960         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4961             "# of rx frames in this range",
4962             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4963         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4964             "# of rx frames in this range",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4966         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4967             "# of rx frames in this range",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4969         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4970             "# of rx frames in this range",
4971             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4972         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4973             "# of rx frames in this range",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4982             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4983         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4992             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4993
4994 #undef SYSCTL_ADD_T4_REG64
4995
4996 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4997         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4998             &pi->stats.name, desc)
4999
5000         /* We get these from port_stats and they may be stale by upto 1s */
5001         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5002             "# drops due to buffer-group 0 overflows");
5003         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5004             "# drops due to buffer-group 1 overflows");
5005         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5006             "# drops due to buffer-group 2 overflows");
5007         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5008             "# drops due to buffer-group 3 overflows");
5009         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5010             "# of buffer-group 0 truncated packets");
5011         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5012             "# of buffer-group 1 truncated packets");
5013         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5014             "# of buffer-group 2 truncated packets");
5015         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5016             "# of buffer-group 3 truncated packets");
5017
5018 #undef SYSCTL_ADD_T4_PORTSTAT
5019
5020         return (0);
5021 }
5022
5023 static int
5024 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5025 {
5026         int rc, *i, space = 0;
5027         struct sbuf sb;
5028
5029         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5030         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5031                 if (space)
5032                         sbuf_printf(&sb, " ");
5033                 sbuf_printf(&sb, "%d", *i);
5034                 space = 1;
5035         }
5036         sbuf_finish(&sb);
5037         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5038         sbuf_delete(&sb);
5039         return (rc);
5040 }
5041
5042 static int
5043 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5044 {
5045         int rc;
5046         struct sbuf *sb;
5047
5048         rc = sysctl_wire_old_buffer(req, 0);
5049         if (rc != 0)
5050                 return(rc);
5051
5052         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5053         if (sb == NULL)
5054                 return (ENOMEM);
5055
5056         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5057         rc = sbuf_finish(sb);
5058         sbuf_delete(sb);
5059
5060         return (rc);
5061 }
5062
5063 static int
5064 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5065 {
5066         struct port_info *pi = arg1;
5067         int op = arg2;
5068         struct adapter *sc = pi->adapter;
5069         u_int v;
5070         int rc;
5071
5072         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5073         if (rc)
5074                 return (rc);
5075         /* XXX: magic numbers */
5076         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5077             &v);
5078         end_synchronized_op(sc, 0);
5079         if (rc)
5080                 return (rc);
5081         if (op == 0)
5082                 v /= 256;
5083
5084         rc = sysctl_handle_int(oidp, &v, 0, req);
5085         return (rc);
5086 }
5087
5088 static int
5089 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5090 {
5091         struct port_info *pi = arg1;
5092         int rc, val;
5093
5094         val = pi->rsrv_noflowq;
5095         rc = sysctl_handle_int(oidp, &val, 0, req);
5096         if (rc != 0 || req->newptr == NULL)
5097                 return (rc);
5098
5099         if ((val >= 1) && (pi->ntxq > 1))
5100                 pi->rsrv_noflowq = 1;
5101         else
5102                 pi->rsrv_noflowq = 0;
5103
5104         return (rc);
5105 }
5106
5107 static int
5108 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5109 {
5110         struct port_info *pi = arg1;
5111         struct adapter *sc = pi->adapter;
5112         int idx, rc, i;
5113         struct sge_rxq *rxq;
5114 #ifdef TCP_OFFLOAD
5115         struct sge_ofld_rxq *ofld_rxq;
5116 #endif
5117         uint8_t v;
5118
5119         idx = pi->tmr_idx;
5120
5121         rc = sysctl_handle_int(oidp, &idx, 0, req);
5122         if (rc != 0 || req->newptr == NULL)
5123                 return (rc);
5124
5125         if (idx < 0 || idx >= SGE_NTIMERS)
5126                 return (EINVAL);
5127
5128         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5129             "t4tmr");
5130         if (rc)
5131                 return (rc);
5132
5133         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5134         for_each_rxq(pi, i, rxq) {
5135 #ifdef atomic_store_rel_8
5136                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5137 #else
5138                 rxq->iq.intr_params = v;
5139 #endif
5140         }
5141 #ifdef TCP_OFFLOAD
5142         for_each_ofld_rxq(pi, i, ofld_rxq) {
5143 #ifdef atomic_store_rel_8
5144                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5145 #else
5146                 ofld_rxq->iq.intr_params = v;
5147 #endif
5148         }
5149 #endif
5150         pi->tmr_idx = idx;
5151
5152         end_synchronized_op(sc, LOCK_HELD);
5153         return (0);
5154 }
5155
5156 static int
5157 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5158 {
5159         struct port_info *pi = arg1;
5160         struct adapter *sc = pi->adapter;
5161         int idx, rc;
5162
5163         idx = pi->pktc_idx;
5164
5165         rc = sysctl_handle_int(oidp, &idx, 0, req);
5166         if (rc != 0 || req->newptr == NULL)
5167                 return (rc);
5168
5169         if (idx < -1 || idx >= SGE_NCOUNTERS)
5170                 return (EINVAL);
5171
5172         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5173             "t4pktc");
5174         if (rc)
5175                 return (rc);
5176
5177         if (pi->flags & PORT_INIT_DONE)
5178                 rc = EBUSY; /* cannot be changed once the queues are created */
5179         else
5180                 pi->pktc_idx = idx;
5181
5182         end_synchronized_op(sc, LOCK_HELD);
5183         return (rc);
5184 }
5185
5186 static int
5187 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5188 {
5189         struct port_info *pi = arg1;
5190         struct adapter *sc = pi->adapter;
5191         int qsize, rc;
5192
5193         qsize = pi->qsize_rxq;
5194
5195         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5196         if (rc != 0 || req->newptr == NULL)
5197                 return (rc);
5198
5199         if (qsize < 128 || (qsize & 7))
5200                 return (EINVAL);
5201
5202         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5203             "t4rxqs");
5204         if (rc)
5205                 return (rc);
5206
5207         if (pi->flags & PORT_INIT_DONE)
5208                 rc = EBUSY; /* cannot be changed once the queues are created */
5209         else
5210                 pi->qsize_rxq = qsize;
5211
5212         end_synchronized_op(sc, LOCK_HELD);
5213         return (rc);
5214 }
5215
5216 static int
5217 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5218 {
5219         struct port_info *pi = arg1;
5220         struct adapter *sc = pi->adapter;
5221         int qsize, rc;
5222
5223         qsize = pi->qsize_txq;
5224
5225         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5226         if (rc != 0 || req->newptr == NULL)
5227                 return (rc);
5228
5229         if (qsize < 128 || qsize > 65536)
5230                 return (EINVAL);
5231
5232         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5233             "t4txqs");
5234         if (rc)
5235                 return (rc);
5236
5237         if (pi->flags & PORT_INIT_DONE)
5238                 rc = EBUSY; /* cannot be changed once the queues are created */
5239         else
5240                 pi->qsize_txq = qsize;
5241
5242         end_synchronized_op(sc, LOCK_HELD);
5243         return (rc);
5244 }
5245
5246 static int
5247 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5248 {
5249         struct port_info *pi = arg1;
5250         struct adapter *sc = pi->adapter;
5251         struct link_config *lc = &pi->link_cfg;
5252         int rc;
5253
5254         if (req->newptr == NULL) {
5255                 struct sbuf *sb;
5256                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5257
5258                 rc = sysctl_wire_old_buffer(req, 0);
5259                 if (rc != 0)
5260                         return(rc);
5261
5262                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5263                 if (sb == NULL)
5264                         return (ENOMEM);
5265
5266                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5267                 rc = sbuf_finish(sb);
5268                 sbuf_delete(sb);
5269         } else {
5270                 char s[2];
5271                 int n;
5272
5273                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5274                 s[1] = 0;
5275
5276                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5277                 if (rc != 0)
5278                         return(rc);
5279
5280                 if (s[1] != 0)
5281                         return (EINVAL);
5282                 if (s[0] < '0' || s[0] > '9')
5283                         return (EINVAL);        /* not a number */
5284                 n = s[0] - '0';
5285                 if (n & ~(PAUSE_TX | PAUSE_RX))
5286                         return (EINVAL);        /* some other bit is set too */
5287
5288                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5289                 if (rc)
5290                         return (rc);
5291                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5292                         int link_ok = lc->link_ok;
5293
5294                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5295                         lc->requested_fc |= n;
5296                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5297                         lc->link_ok = link_ok;  /* restore */
5298                 }
5299                 end_synchronized_op(sc, 0);
5300         }
5301
5302         return (rc);
5303 }
5304
5305 static int
5306 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5307 {
5308         struct adapter *sc = arg1;
5309         int reg = arg2;
5310         uint64_t val;
5311
5312         val = t4_read_reg64(sc, reg);
5313
5314         return (sysctl_handle_64(oidp, &val, 0, req));
5315 }
5316
5317 static int
5318 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5319 {
5320         struct adapter *sc = arg1;
5321         int rc, t;
5322         uint32_t param, val;
5323
5324         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5325         if (rc)
5326                 return (rc);
5327         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5328             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5329             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5330         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5331         end_synchronized_op(sc, 0);
5332         if (rc)
5333                 return (rc);
5334
5335         /* unknown is returned as 0 but we display -1 in that case */
5336         t = val == 0 ? -1 : val;
5337
5338         rc = sysctl_handle_int(oidp, &t, 0, req);
5339         return (rc);
5340 }
5341
5342 #ifdef SBUF_DRAIN
5343 static int
5344 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5345 {
5346         struct adapter *sc = arg1;
5347         struct sbuf *sb;
5348         int rc, i;
5349         uint16_t incr[NMTUS][NCCTRL_WIN];
5350         static const char *dec_fac[] = {
5351                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5352                 "0.9375"
5353         };
5354
5355         rc = sysctl_wire_old_buffer(req, 0);
5356         if (rc != 0)
5357                 return (rc);
5358
5359         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5360         if (sb == NULL)
5361                 return (ENOMEM);
5362
5363         t4_read_cong_tbl(sc, incr);
5364
5365         for (i = 0; i < NCCTRL_WIN; ++i) {
5366                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5367                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5368                     incr[5][i], incr[6][i], incr[7][i]);
5369                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5370                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5371                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5372                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5373         }
5374
5375         rc = sbuf_finish(sb);
5376         sbuf_delete(sb);
5377
5378         return (rc);
5379 }
5380
5381 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5382         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5383         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5384         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5385 };
5386
5387 static int
5388 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5389 {
5390         struct adapter *sc = arg1;
5391         struct sbuf *sb;
5392         int rc, i, n, qid = arg2;
5393         uint32_t *buf, *p;
5394         char *qtype;
5395         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5396
5397         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5398             ("%s: bad qid %d\n", __func__, qid));
5399
5400         if (qid < CIM_NUM_IBQ) {
5401                 /* inbound queue */
5402                 qtype = "IBQ";
5403                 n = 4 * CIM_IBQ_SIZE;
5404                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5405                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5406         } else {
5407                 /* outbound queue */
5408                 qtype = "OBQ";
5409                 qid -= CIM_NUM_IBQ;
5410                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5411                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5412                 rc = t4_read_cim_obq(sc, qid, buf, n);
5413         }
5414
5415         if (rc < 0) {
5416                 rc = -rc;
5417                 goto done;
5418         }
5419         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5420
5421         rc = sysctl_wire_old_buffer(req, 0);
5422         if (rc != 0)
5423                 goto done;
5424
5425         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5426         if (sb == NULL) {
5427                 rc = ENOMEM;
5428                 goto done;
5429         }
5430
5431         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5432         for (i = 0, p = buf; i < n; i += 16, p += 4)
5433                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5434                     p[2], p[3]);
5435
5436         rc = sbuf_finish(sb);
5437         sbuf_delete(sb);
5438 done:
5439         free(buf, M_CXGBE);
5440         return (rc);
5441 }
5442
5443 static int
5444 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5445 {
5446         struct adapter *sc = arg1;
5447         u_int cfg;
5448         struct sbuf *sb;
5449         uint32_t *buf, *p;
5450         int rc;
5451
5452         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5453         if (rc != 0)
5454                 return (rc);
5455
5456         rc = sysctl_wire_old_buffer(req, 0);
5457         if (rc != 0)
5458                 return (rc);
5459
5460         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5461         if (sb == NULL)
5462                 return (ENOMEM);
5463
5464         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5465             M_ZERO | M_WAITOK);
5466
5467         rc = -t4_cim_read_la(sc, buf, NULL);
5468         if (rc != 0)
5469                 goto done;
5470
5471         sbuf_printf(sb, "Status   Data      PC%s",
5472             cfg & F_UPDBGLACAPTPCONLY ? "" :
5473             "     LS0Stat  LS0Addr             LS0Data");
5474
5475         KASSERT((sc->params.cim_la_size & 7) == 0,
5476             ("%s: p will walk off the end of buf", __func__));
5477
5478         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5479                 if (cfg & F_UPDBGLACAPTPCONLY) {
5480                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5481                             p[6], p[7]);
5482                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5483                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5484                             p[4] & 0xff, p[5] >> 8);
5485                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5486                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5487                             p[1] & 0xf, p[2] >> 4);
5488                 } else {
5489                         sbuf_printf(sb,
5490                             "\n  %02x   %x%07x %x%07x %08x %08x "
5491                             "%08x%08x%08x%08x",
5492                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5493                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5494                             p[6], p[7]);
5495                 }
5496         }
5497
5498         rc = sbuf_finish(sb);
5499         sbuf_delete(sb);
5500 done:
5501         free(buf, M_CXGBE);
5502         return (rc);
5503 }
5504
5505 static int
5506 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5507 {
5508         struct adapter *sc = arg1;
5509         u_int i;
5510         struct sbuf *sb;
5511         uint32_t *buf, *p;
5512         int rc;
5513
5514         rc = sysctl_wire_old_buffer(req, 0);
5515         if (rc != 0)
5516                 return (rc);
5517
5518         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5519         if (sb == NULL)
5520                 return (ENOMEM);
5521
5522         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5523             M_ZERO | M_WAITOK);
5524
5525         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5526         p = buf;
5527
5528         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5529                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5530                     p[1], p[0]);
5531         }
5532
5533         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5534         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5535                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5536                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5537                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5538                     (p[1] >> 2) | ((p[2] & 3) << 30),
5539                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5540                     p[0] & 1);
5541         }
5542
5543         rc = sbuf_finish(sb);
5544         sbuf_delete(sb);
5545         free(buf, M_CXGBE);
5546         return (rc);
5547 }
5548
5549 static int
5550 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5551 {
5552         struct adapter *sc = arg1;
5553         u_int i;
5554         struct sbuf *sb;
5555         uint32_t *buf, *p;
5556         int rc;
5557
5558         rc = sysctl_wire_old_buffer(req, 0);
5559         if (rc != 0)
5560                 return (rc);
5561
5562         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5563         if (sb == NULL)
5564                 return (ENOMEM);
5565
5566         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5567             M_ZERO | M_WAITOK);
5568
5569         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5570         p = buf;
5571
5572         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5573         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5574                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5575                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5576                     p[4], p[3], p[2], p[1], p[0]);
5577         }
5578
5579         sbuf_printf(sb, "\n\nCntl ID               Data");
5580         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5581                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5582                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5583         }
5584
5585         rc = sbuf_finish(sb);
5586         sbuf_delete(sb);
5587         free(buf, M_CXGBE);
5588         return (rc);
5589 }
5590
5591 static int
5592 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5593 {
5594         struct adapter *sc = arg1;
5595         struct sbuf *sb;
5596         int rc, i;
5597         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5598         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5599         uint16_t thres[CIM_NUM_IBQ];
5600         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5601         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5602         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5603
5604         if (is_t4(sc)) {
5605                 cim_num_obq = CIM_NUM_OBQ;
5606                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5607                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5608         } else {
5609                 cim_num_obq = CIM_NUM_OBQ_T5;
5610                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5611                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5612         }
5613         nq = CIM_NUM_IBQ + cim_num_obq;
5614
5615         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5616         if (rc == 0)
5617                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5618         if (rc != 0)
5619                 return (rc);
5620
5621         t4_read_cimq_cfg(sc, base, size, thres);
5622
5623         rc = sysctl_wire_old_buffer(req, 0);
5624         if (rc != 0)
5625                 return (rc);
5626
5627         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5628         if (sb == NULL)
5629                 return (ENOMEM);
5630
5631         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5632
5633         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5634                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5635                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5636                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5637                     G_QUEREMFLITS(p[2]) * 16);
5638         for ( ; i < nq; i++, p += 4, wr += 2)
5639                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5640                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5641                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5642                     G_QUEREMFLITS(p[2]) * 16);
5643
5644         rc = sbuf_finish(sb);
5645         sbuf_delete(sb);
5646
5647         return (rc);
5648 }
5649
5650 static int
5651 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5652 {
5653         struct adapter *sc = arg1;
5654         struct sbuf *sb;
5655         int rc;
5656         struct tp_cpl_stats stats;
5657
5658         rc = sysctl_wire_old_buffer(req, 0);
5659         if (rc != 0)
5660                 return (rc);
5661
5662         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5663         if (sb == NULL)
5664                 return (ENOMEM);
5665
5666         t4_tp_get_cpl_stats(sc, &stats);
5667
5668         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5669             "channel 3\n");
5670         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5671                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5672         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5673                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5674
5675         rc = sbuf_finish(sb);
5676         sbuf_delete(sb);
5677
5678         return (rc);
5679 }
5680
5681 static int
5682 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5683 {
5684         struct adapter *sc = arg1;
5685         struct sbuf *sb;
5686         int rc;
5687         struct tp_usm_stats stats;
5688
5689         rc = sysctl_wire_old_buffer(req, 0);
5690         if (rc != 0)
5691                 return(rc);
5692
5693         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5694         if (sb == NULL)
5695                 return (ENOMEM);
5696
5697         t4_get_usm_stats(sc, &stats);
5698
5699         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5700         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5701         sbuf_printf(sb, "Drops:  %u", stats.drops);
5702
5703         rc = sbuf_finish(sb);
5704         sbuf_delete(sb);
5705
5706         return (rc);
5707 }
5708
5709 const char *devlog_level_strings[] = {
5710         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5711         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5712         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5713         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5714         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5715         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5716 };
5717
5718 const char *devlog_facility_strings[] = {
5719         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5720         [FW_DEVLOG_FACILITY_CF]         = "CF",
5721         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5722         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5723         [FW_DEVLOG_FACILITY_RES]        = "RES",
5724         [FW_DEVLOG_FACILITY_HW]         = "HW",
5725         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5726         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5727         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5728         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5729         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5730         [FW_DEVLOG_FACILITY_VI]         = "VI",
5731         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5732         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5733         [FW_DEVLOG_FACILITY_TM]         = "TM",
5734         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5735         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5736         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5737         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5738         [FW_DEVLOG_FACILITY_RI]         = "RI",
5739         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5740         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5741         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5742         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5743 };
5744
5745 static int
5746 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5747 {
5748         struct adapter *sc = arg1;
5749         struct devlog_params *dparams = &sc->params.devlog;
5750         struct fw_devlog_e *buf, *e;
5751         int i, j, rc, nentries, first = 0, m;
5752         struct sbuf *sb;
5753         uint64_t ftstamp = UINT64_MAX;
5754
5755         if (dparams->start == 0) {
5756                 dparams->memtype = FW_MEMTYPE_EDC0;
5757                 dparams->start = 0x84000;
5758                 dparams->size = 32768;
5759         }
5760
5761         nentries = dparams->size / sizeof(struct fw_devlog_e);
5762
5763         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5764         if (buf == NULL)
5765                 return (ENOMEM);
5766
5767         m = fwmtype_to_hwmtype(dparams->memtype);
5768         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5769         if (rc != 0)
5770                 goto done;
5771
5772         for (i = 0; i < nentries; i++) {
5773                 e = &buf[i];
5774
5775                 if (e->timestamp == 0)
5776                         break;  /* end */
5777
5778                 e->timestamp = be64toh(e->timestamp);
5779                 e->seqno = be32toh(e->seqno);
5780                 for (j = 0; j < 8; j++)
5781                         e->params[j] = be32toh(e->params[j]);
5782
5783                 if (e->timestamp < ftstamp) {
5784                         ftstamp = e->timestamp;
5785                         first = i;
5786                 }
5787         }
5788
5789         if (buf[first].timestamp == 0)
5790                 goto done;      /* nothing in the log */
5791
5792         rc = sysctl_wire_old_buffer(req, 0);
5793         if (rc != 0)
5794                 goto done;
5795
5796         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5797         if (sb == NULL) {
5798                 rc = ENOMEM;
5799                 goto done;
5800         }
5801         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5802             "Seq#", "Tstamp", "Level", "Facility", "Message");
5803
5804         i = first;
5805         do {
5806                 e = &buf[i];
5807                 if (e->timestamp == 0)
5808                         break;  /* end */
5809
5810                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5811                     e->seqno, e->timestamp,
5812                     (e->level < nitems(devlog_level_strings) ?
5813                         devlog_level_strings[e->level] : "UNKNOWN"),
5814                     (e->facility < nitems(devlog_facility_strings) ?
5815                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5816                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5817                     e->params[2], e->params[3], e->params[4],
5818                     e->params[5], e->params[6], e->params[7]);
5819
5820                 if (++i == nentries)
5821                         i = 0;
5822         } while (i != first);
5823
5824         rc = sbuf_finish(sb);
5825         sbuf_delete(sb);
5826 done:
5827         free(buf, M_CXGBE);
5828         return (rc);
5829 }
5830
5831 static int
5832 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5833 {
5834         struct adapter *sc = arg1;
5835         struct sbuf *sb;
5836         int rc;
5837         struct tp_fcoe_stats stats[4];
5838
5839         rc = sysctl_wire_old_buffer(req, 0);
5840         if (rc != 0)
5841                 return (rc);
5842
5843         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5844         if (sb == NULL)
5845                 return (ENOMEM);
5846
5847         t4_get_fcoe_stats(sc, 0, &stats[0]);
5848         t4_get_fcoe_stats(sc, 1, &stats[1]);
5849         t4_get_fcoe_stats(sc, 2, &stats[2]);
5850         t4_get_fcoe_stats(sc, 3, &stats[3]);
5851
5852         sbuf_printf(sb, "                   channel 0        channel 1        "
5853             "channel 2        channel 3\n");
5854         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5855             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5856             stats[3].octetsDDP);
5857         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5858             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5859         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5860             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5861             stats[3].framesDrop);
5862
5863         rc = sbuf_finish(sb);
5864         sbuf_delete(sb);
5865
5866         return (rc);
5867 }
5868
5869 static int
5870 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5871 {
5872         struct adapter *sc = arg1;
5873         struct sbuf *sb;
5874         int rc, i;
5875         unsigned int map, kbps, ipg, mode;
5876         unsigned int pace_tab[NTX_SCHED];
5877
5878         rc = sysctl_wire_old_buffer(req, 0);
5879         if (rc != 0)
5880                 return (rc);
5881
5882         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5883         if (sb == NULL)
5884                 return (ENOMEM);
5885
5886         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5887         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5888         t4_read_pace_tbl(sc, pace_tab);
5889
5890         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5891             "Class IPG (0.1 ns)   Flow IPG (us)");
5892
5893         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5894                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5895                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5896                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5897                 if (kbps)
5898                         sbuf_printf(sb, "%9u     ", kbps);
5899                 else
5900                         sbuf_printf(sb, " disabled     ");
5901
5902                 if (ipg)
5903                         sbuf_printf(sb, "%13u        ", ipg);
5904                 else
5905                         sbuf_printf(sb, "     disabled        ");
5906
5907                 if (pace_tab[i])
5908                         sbuf_printf(sb, "%10u", pace_tab[i]);
5909                 else
5910                         sbuf_printf(sb, "  disabled");
5911         }
5912
5913         rc = sbuf_finish(sb);
5914         sbuf_delete(sb);
5915
5916         return (rc);
5917 }
5918
5919 static int
5920 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5921 {
5922         struct adapter *sc = arg1;
5923         struct sbuf *sb;
5924         int rc, i, j;
5925         uint64_t *p0, *p1;
5926         struct lb_port_stats s[2];
5927         static const char *stat_name[] = {
5928                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5929                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5930                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5931                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5932                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5933                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5934                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5935         };
5936
5937         rc = sysctl_wire_old_buffer(req, 0);
5938         if (rc != 0)
5939                 return (rc);
5940
5941         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5942         if (sb == NULL)
5943                 return (ENOMEM);
5944
5945         memset(s, 0, sizeof(s));
5946
5947         for (i = 0; i < 4; i += 2) {
5948                 t4_get_lb_stats(sc, i, &s[0]);
5949                 t4_get_lb_stats(sc, i + 1, &s[1]);
5950
5951                 p0 = &s[0].octets;
5952                 p1 = &s[1].octets;
5953                 sbuf_printf(sb, "%s                       Loopback %u"
5954                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5955
5956                 for (j = 0; j < nitems(stat_name); j++)
5957                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5958                                    *p0++, *p1++);
5959         }
5960
5961         rc = sbuf_finish(sb);
5962         sbuf_delete(sb);
5963
5964         return (rc);
5965 }
5966
5967 static int
5968 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5969 {
5970         int rc = 0;
5971         struct port_info *pi = arg1;
5972         struct sbuf *sb;
5973         static const char *linkdnreasons[] = {
5974                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5975                 "PHY overheated", "unknown", "rx los", "reserved7"
5976         };
5977
5978         rc = sysctl_wire_old_buffer(req, 0);
5979         if (rc != 0)
5980                 return(rc);
5981         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5982         if (sb == NULL)
5983                 return (ENOMEM);
5984
5985         if (pi->linkdnrc < 0)
5986                 sbuf_printf(sb, "n/a");
5987         else if (pi->linkdnrc < nitems(linkdnreasons))
5988                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5989         else
5990                 sbuf_printf(sb, "%d", pi->linkdnrc);
5991
5992         rc = sbuf_finish(sb);
5993         sbuf_delete(sb);
5994
5995         return (rc);
5996 }
5997
5998 struct mem_desc {
5999         unsigned int base;
6000         unsigned int limit;
6001         unsigned int idx;
6002 };
6003
6004 static int
6005 mem_desc_cmp(const void *a, const void *b)
6006 {
6007         return ((const struct mem_desc *)a)->base -
6008                ((const struct mem_desc *)b)->base;
6009 }
6010
6011 static void
6012 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6013     unsigned int to)
6014 {
6015         unsigned int size;
6016
6017         size = to - from + 1;
6018         if (size == 0)
6019                 return;
6020
6021         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6022         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6023 }
6024
6025 static int
6026 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6027 {
6028         struct adapter *sc = arg1;
6029         struct sbuf *sb;
6030         int rc, i, n;
6031         uint32_t lo, hi, used, alloc;
6032         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6033         static const char *region[] = {
6034                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6035                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6036                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6037                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6038                 "RQUDP region:", "PBL region:", "TXPBL region:",
6039                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6040                 "On-chip queues:"
6041         };
6042         struct mem_desc avail[4];
6043         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6044         struct mem_desc *md = mem;
6045
6046         rc = sysctl_wire_old_buffer(req, 0);
6047         if (rc != 0)
6048                 return (rc);
6049
6050         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6051         if (sb == NULL)
6052                 return (ENOMEM);
6053
6054         for (i = 0; i < nitems(mem); i++) {
6055                 mem[i].limit = 0;
6056                 mem[i].idx = i;
6057         }
6058
6059         /* Find and sort the populated memory ranges */
6060         i = 0;
6061         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6062         if (lo & F_EDRAM0_ENABLE) {
6063                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6064                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6065                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6066                 avail[i].idx = 0;
6067                 i++;
6068         }
6069         if (lo & F_EDRAM1_ENABLE) {
6070                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6071                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6072                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6073                 avail[i].idx = 1;
6074                 i++;
6075         }
6076         if (lo & F_EXT_MEM_ENABLE) {
6077                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6078                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6079                 avail[i].limit = avail[i].base +
6080                     (G_EXT_MEM_SIZE(hi) << 20);
6081                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6082                 i++;
6083         }
6084         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6085                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6086                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6087                 avail[i].limit = avail[i].base +
6088                     (G_EXT_MEM1_SIZE(hi) << 20);
6089                 avail[i].idx = 4;
6090                 i++;
6091         }
6092         if (!i)                                    /* no memory available */
6093                 return 0;
6094         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6095
6096         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6097         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6098         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6099         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6100         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6101         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6102         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6103         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6104         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6105
6106         /* the next few have explicit upper bounds */
6107         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6108         md->limit = md->base - 1 +
6109                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6110                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6111         md++;
6112
6113         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6114         md->limit = md->base - 1 +
6115                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6116                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6117         md++;
6118
6119         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6120                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6121                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6122                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6123         } else {
6124                 md->base = 0;
6125                 md->idx = nitems(region);  /* hide it */
6126         }
6127         md++;
6128
6129 #define ulp_region(reg) \
6130         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6131         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6132
6133         ulp_region(RX_ISCSI);
6134         ulp_region(RX_TDDP);
6135         ulp_region(TX_TPT);
6136         ulp_region(RX_STAG);
6137         ulp_region(RX_RQ);
6138         ulp_region(RX_RQUDP);
6139         ulp_region(RX_PBL);
6140         ulp_region(TX_PBL);
6141 #undef ulp_region
6142
6143         md->base = 0;
6144         md->idx = nitems(region);
6145         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6146                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6147                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6148                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6149         }
6150         md++;
6151
6152         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6153         md->limit = md->base + sc->tids.ntids - 1;
6154         md++;
6155         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6156         md->limit = md->base + sc->tids.ntids - 1;
6157         md++;
6158
6159         md->base = sc->vres.ocq.start;
6160         if (sc->vres.ocq.size)
6161                 md->limit = md->base + sc->vres.ocq.size - 1;
6162         else
6163                 md->idx = nitems(region);  /* hide it */
6164         md++;
6165
6166         /* add any address-space holes, there can be up to 3 */
6167         for (n = 0; n < i - 1; n++)
6168                 if (avail[n].limit < avail[n + 1].base)
6169                         (md++)->base = avail[n].limit;
6170         if (avail[n].limit)
6171                 (md++)->base = avail[n].limit;
6172
6173         n = md - mem;
6174         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6175
6176         for (lo = 0; lo < i; lo++)
6177                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6178                                 avail[lo].limit - 1);
6179
6180         sbuf_printf(sb, "\n");
6181         for (i = 0; i < n; i++) {
6182                 if (mem[i].idx >= nitems(region))
6183                         continue;                        /* skip holes */
6184                 if (!mem[i].limit)
6185                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6186                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6187                                 mem[i].limit);
6188         }
6189
6190         sbuf_printf(sb, "\n");
6191         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6192         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6193         mem_region_show(sb, "uP RAM:", lo, hi);
6194
6195         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6196         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6197         mem_region_show(sb, "uP Extmem2:", lo, hi);
6198
6199         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6200         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6201                    G_PMRXMAXPAGE(lo),
6202                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6203                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6204
6205         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6206         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6207         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6208                    G_PMTXMAXPAGE(lo),
6209                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6210                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6211         sbuf_printf(sb, "%u p-structs\n",
6212                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6213
6214         for (i = 0; i < 4; i++) {
6215                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6216                 if (is_t4(sc)) {
6217                         used = G_USED(lo);
6218                         alloc = G_ALLOC(lo);
6219                 } else {
6220                         used = G_T5_USED(lo);
6221                         alloc = G_T5_ALLOC(lo);
6222                 }
6223                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6224                            i, used, alloc);
6225         }
6226         for (i = 0; i < 4; i++) {
6227                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6228                 if (is_t4(sc)) {
6229                         used = G_USED(lo);
6230                         alloc = G_ALLOC(lo);
6231                 } else {
6232                         used = G_T5_USED(lo);
6233                         alloc = G_T5_ALLOC(lo);
6234                 }
6235                 sbuf_printf(sb,
6236                            "\nLoopback %d using %u pages out of %u allocated",
6237                            i, used, alloc);
6238         }
6239
6240         rc = sbuf_finish(sb);
6241         sbuf_delete(sb);
6242
6243         return (rc);
6244 }
6245
6246 static inline void
6247 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6248 {
6249         *mask = x | y;
6250         y = htobe64(y);
6251         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6252 }
6253
6254 static int
6255 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6256 {
6257         struct adapter *sc = arg1;
6258         struct sbuf *sb;
6259         int rc, i, n;
6260
6261         rc = sysctl_wire_old_buffer(req, 0);
6262         if (rc != 0)
6263                 return (rc);
6264
6265         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6266         if (sb == NULL)
6267                 return (ENOMEM);
6268
6269         sbuf_printf(sb,
6270             "Idx  Ethernet address     Mask     Vld Ports PF"
6271             "  VF              Replication             P0 P1 P2 P3  ML");
6272         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6273             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6274         for (i = 0; i < n; i++) {
6275                 uint64_t tcamx, tcamy, mask;
6276                 uint32_t cls_lo, cls_hi;
6277                 uint8_t addr[ETHER_ADDR_LEN];
6278
6279                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6280                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6281                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6282                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6283
6284                 if (tcamx & tcamy)
6285                         continue;
6286
6287                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6288                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6289                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6290                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6291                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6292                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6293                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6294
6295                 if (cls_lo & F_REPLICATE) {
6296                         struct fw_ldst_cmd ldst_cmd;
6297
6298                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6299                         ldst_cmd.op_to_addrspace =
6300                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6301                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6302                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6303                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6304                         ldst_cmd.u.mps.rplc.fid_idx =
6305                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6306                                 V_FW_LDST_CMD_IDX(i));
6307
6308                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6309                             "t4mps");
6310                         if (rc)
6311                                 break;
6312                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6313                             sizeof(ldst_cmd), &ldst_cmd);
6314                         end_synchronized_op(sc, 0);
6315
6316                         if (rc != 0) {
6317                                 sbuf_printf(sb,
6318                                     " ------------ error %3u ------------", rc);
6319                                 rc = 0;
6320                         } else {
6321                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6322                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6323                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6324                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6325                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6326                         }
6327                 } else
6328                         sbuf_printf(sb, "%36s", "");
6329
6330                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6331                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6332                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6333         }
6334
6335         if (rc)
6336                 (void) sbuf_finish(sb);
6337         else
6338                 rc = sbuf_finish(sb);
6339         sbuf_delete(sb);
6340
6341         return (rc);
6342 }
6343
6344 static int
6345 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6346 {
6347         struct adapter *sc = arg1;
6348         struct sbuf *sb;
6349         int rc;
6350         uint16_t mtus[NMTUS];
6351
6352         rc = sysctl_wire_old_buffer(req, 0);
6353         if (rc != 0)
6354                 return (rc);
6355
6356         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6357         if (sb == NULL)
6358                 return (ENOMEM);
6359
6360         t4_read_mtu_tbl(sc, mtus, NULL);
6361
6362         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6363             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6364             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6365             mtus[14], mtus[15]);
6366
6367         rc = sbuf_finish(sb);
6368         sbuf_delete(sb);
6369
6370         return (rc);
6371 }
6372
6373 static int
6374 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6375 {
6376         struct adapter *sc = arg1;
6377         struct sbuf *sb;
6378         int rc, i;
6379         uint32_t cnt[PM_NSTATS];
6380         uint64_t cyc[PM_NSTATS];
6381         static const char *rx_stats[] = {
6382                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6383         };
6384         static const char *tx_stats[] = {
6385                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6386         };
6387
6388         rc = sysctl_wire_old_buffer(req, 0);
6389         if (rc != 0)
6390                 return (rc);
6391
6392         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6393         if (sb == NULL)
6394                 return (ENOMEM);
6395
6396         t4_pmtx_get_stats(sc, cnt, cyc);
6397         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6398         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6399                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6400                     cyc[i]);
6401
6402         t4_pmrx_get_stats(sc, cnt, cyc);
6403         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6404         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6405                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6406                     cyc[i]);
6407
6408         rc = sbuf_finish(sb);
6409         sbuf_delete(sb);
6410
6411         return (rc);
6412 }
6413
6414 static int
6415 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6416 {
6417         struct adapter *sc = arg1;
6418         struct sbuf *sb;
6419         int rc;
6420         struct tp_rdma_stats stats;
6421
6422         rc = sysctl_wire_old_buffer(req, 0);
6423         if (rc != 0)
6424                 return (rc);
6425
6426         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6427         if (sb == NULL)
6428                 return (ENOMEM);
6429
6430         t4_tp_get_rdma_stats(sc, &stats);
6431         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6432         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6433
6434         rc = sbuf_finish(sb);
6435         sbuf_delete(sb);
6436
6437         return (rc);
6438 }
6439
6440 static int
6441 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6442 {
6443         struct adapter *sc = arg1;
6444         struct sbuf *sb;
6445         int rc;
6446         struct tp_tcp_stats v4, v6;
6447
6448         rc = sysctl_wire_old_buffer(req, 0);
6449         if (rc != 0)
6450                 return (rc);
6451
6452         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6453         if (sb == NULL)
6454                 return (ENOMEM);
6455
6456         t4_tp_get_tcp_stats(sc, &v4, &v6);
6457         sbuf_printf(sb,
6458             "                                IP                 IPv6\n");
6459         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6460             v4.tcpOutRsts, v6.tcpOutRsts);
6461         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6462             v4.tcpInSegs, v6.tcpInSegs);
6463         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6464             v4.tcpOutSegs, v6.tcpOutSegs);
6465         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6466             v4.tcpRetransSegs, v6.tcpRetransSegs);
6467
6468         rc = sbuf_finish(sb);
6469         sbuf_delete(sb);
6470
6471         return (rc);
6472 }
6473
6474 static int
6475 sysctl_tids(SYSCTL_HANDLER_ARGS)
6476 {
6477         struct adapter *sc = arg1;
6478         struct sbuf *sb;
6479         int rc;
6480         struct tid_info *t = &sc->tids;
6481
6482         rc = sysctl_wire_old_buffer(req, 0);
6483         if (rc != 0)
6484                 return (rc);
6485
6486         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6487         if (sb == NULL)
6488                 return (ENOMEM);
6489
6490         if (t->natids) {
6491                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6492                     t->atids_in_use);
6493         }
6494
6495         if (t->ntids) {
6496                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6497                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6498
6499                         if (b) {
6500                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6501                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6502                                     t->ntids - 1);
6503                         } else {
6504                                 sbuf_printf(sb, "TID range: %u-%u",
6505                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6506                                     t->ntids - 1);
6507                         }
6508                 } else
6509                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6510                 sbuf_printf(sb, ", in use: %u\n",
6511                     atomic_load_acq_int(&t->tids_in_use));
6512         }
6513
6514         if (t->nstids) {
6515                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6516                     t->stid_base + t->nstids - 1, t->stids_in_use);
6517         }
6518
6519         if (t->nftids) {
6520                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6521                     t->ftid_base + t->nftids - 1);
6522         }
6523
6524         if (t->netids) {
6525                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6526                     t->etid_base + t->netids - 1);
6527         }
6528
6529         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6530             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6531             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6532
6533         rc = sbuf_finish(sb);
6534         sbuf_delete(sb);
6535
6536         return (rc);
6537 }
6538
6539 static int
6540 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6541 {
6542         struct adapter *sc = arg1;
6543         struct sbuf *sb;
6544         int rc;
6545         struct tp_err_stats stats;
6546
6547         rc = sysctl_wire_old_buffer(req, 0);
6548         if (rc != 0)
6549                 return (rc);
6550
6551         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6552         if (sb == NULL)
6553                 return (ENOMEM);
6554
6555         t4_tp_get_err_stats(sc, &stats);
6556
6557         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6558                       "channel 3\n");
6559         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6560             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6561             stats.macInErrs[3]);
6562         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6563             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6564             stats.hdrInErrs[3]);
6565         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6566             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6567             stats.tcpInErrs[3]);
6568         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6569             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6570             stats.tcp6InErrs[3]);
6571         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6572             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6573             stats.tnlCongDrops[3]);
6574         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6575             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6576             stats.tnlTxDrops[3]);
6577         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6578             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6579             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6580         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6581             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6582             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6583         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6584             stats.ofldNoNeigh, stats.ofldCongDefer);
6585
6586         rc = sbuf_finish(sb);
6587         sbuf_delete(sb);
6588
6589         return (rc);
6590 }
6591
6592 struct field_desc {
6593         const char *name;
6594         u_int start;
6595         u_int width;
6596 };
6597
6598 static void
6599 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6600 {
6601         char buf[32];
6602         int line_size = 0;
6603
6604         while (f->name) {
6605                 uint64_t mask = (1ULL << f->width) - 1;
6606                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6607                     ((uintmax_t)v >> f->start) & mask);
6608
6609                 if (line_size + len >= 79) {
6610                         line_size = 8;
6611                         sbuf_printf(sb, "\n        ");
6612                 }
6613                 sbuf_printf(sb, "%s ", buf);
6614                 line_size += len + 1;
6615                 f++;
6616         }
6617         sbuf_printf(sb, "\n");
6618 }
6619
6620 static struct field_desc tp_la0[] = {
6621         { "RcfOpCodeOut", 60, 4 },
6622         { "State", 56, 4 },
6623         { "WcfState", 52, 4 },
6624         { "RcfOpcSrcOut", 50, 2 },
6625         { "CRxError", 49, 1 },
6626         { "ERxError", 48, 1 },
6627         { "SanityFailed", 47, 1 },
6628         { "SpuriousMsg", 46, 1 },
6629         { "FlushInputMsg", 45, 1 },
6630         { "FlushInputCpl", 44, 1 },
6631         { "RssUpBit", 43, 1 },
6632         { "RssFilterHit", 42, 1 },
6633         { "Tid", 32, 10 },
6634         { "InitTcb", 31, 1 },
6635         { "LineNumber", 24, 7 },
6636         { "Emsg", 23, 1 },
6637         { "EdataOut", 22, 1 },
6638         { "Cmsg", 21, 1 },
6639         { "CdataOut", 20, 1 },
6640         { "EreadPdu", 19, 1 },
6641         { "CreadPdu", 18, 1 },
6642         { "TunnelPkt", 17, 1 },
6643         { "RcfPeerFin", 16, 1 },
6644         { "RcfReasonOut", 12, 4 },
6645         { "TxCchannel", 10, 2 },
6646         { "RcfTxChannel", 8, 2 },
6647         { "RxEchannel", 6, 2 },
6648         { "RcfRxChannel", 5, 1 },
6649         { "RcfDataOutSrdy", 4, 1 },
6650         { "RxDvld", 3, 1 },
6651         { "RxOoDvld", 2, 1 },
6652         { "RxCongestion", 1, 1 },
6653         { "TxCongestion", 0, 1 },
6654         { NULL }
6655 };
6656
6657 static struct field_desc tp_la1[] = {
6658         { "CplCmdIn", 56, 8 },
6659         { "CplCmdOut", 48, 8 },
6660         { "ESynOut", 47, 1 },
6661         { "EAckOut", 46, 1 },
6662         { "EFinOut", 45, 1 },
6663         { "ERstOut", 44, 1 },
6664         { "SynIn", 43, 1 },
6665         { "AckIn", 42, 1 },
6666         { "FinIn", 41, 1 },
6667         { "RstIn", 40, 1 },
6668         { "DataIn", 39, 1 },
6669         { "DataInVld", 38, 1 },
6670         { "PadIn", 37, 1 },
6671         { "RxBufEmpty", 36, 1 },
6672         { "RxDdp", 35, 1 },
6673         { "RxFbCongestion", 34, 1 },
6674         { "TxFbCongestion", 33, 1 },
6675         { "TxPktSumSrdy", 32, 1 },
6676         { "RcfUlpType", 28, 4 },
6677         { "Eread", 27, 1 },
6678         { "Ebypass", 26, 1 },
6679         { "Esave", 25, 1 },
6680         { "Static0", 24, 1 },
6681         { "Cread", 23, 1 },
6682         { "Cbypass", 22, 1 },
6683         { "Csave", 21, 1 },
6684         { "CPktOut", 20, 1 },
6685         { "RxPagePoolFull", 18, 2 },
6686         { "RxLpbkPkt", 17, 1 },
6687         { "TxLpbkPkt", 16, 1 },
6688         { "RxVfValid", 15, 1 },
6689         { "SynLearned", 14, 1 },
6690         { "SetDelEntry", 13, 1 },
6691         { "SetInvEntry", 12, 1 },
6692         { "CpcmdDvld", 11, 1 },
6693         { "CpcmdSave", 10, 1 },
6694         { "RxPstructsFull", 8, 2 },
6695         { "EpcmdDvld", 7, 1 },
6696         { "EpcmdFlush", 6, 1 },
6697         { "EpcmdTrimPrefix", 5, 1 },
6698         { "EpcmdTrimPostfix", 4, 1 },
6699         { "ERssIp4Pkt", 3, 1 },
6700         { "ERssIp6Pkt", 2, 1 },
6701         { "ERssTcpUdpPkt", 1, 1 },
6702         { "ERssFceFipPkt", 0, 1 },
6703         { NULL }
6704 };
6705
6706 static struct field_desc tp_la2[] = {
6707         { "CplCmdIn", 56, 8 },
6708         { "MpsVfVld", 55, 1 },
6709         { "MpsPf", 52, 3 },
6710         { "MpsVf", 44, 8 },
6711         { "SynIn", 43, 1 },
6712         { "AckIn", 42, 1 },
6713         { "FinIn", 41, 1 },
6714         { "RstIn", 40, 1 },
6715         { "DataIn", 39, 1 },
6716         { "DataInVld", 38, 1 },
6717         { "PadIn", 37, 1 },
6718         { "RxBufEmpty", 36, 1 },
6719         { "RxDdp", 35, 1 },
6720         { "RxFbCongestion", 34, 1 },
6721         { "TxFbCongestion", 33, 1 },
6722         { "TxPktSumSrdy", 32, 1 },
6723         { "RcfUlpType", 28, 4 },
6724         { "Eread", 27, 1 },
6725         { "Ebypass", 26, 1 },
6726         { "Esave", 25, 1 },
6727         { "Static0", 24, 1 },
6728         { "Cread", 23, 1 },
6729         { "Cbypass", 22, 1 },
6730         { "Csave", 21, 1 },
6731         { "CPktOut", 20, 1 },
6732         { "RxPagePoolFull", 18, 2 },
6733         { "RxLpbkPkt", 17, 1 },
6734         { "TxLpbkPkt", 16, 1 },
6735         { "RxVfValid", 15, 1 },
6736         { "SynLearned", 14, 1 },
6737         { "SetDelEntry", 13, 1 },
6738         { "SetInvEntry", 12, 1 },
6739         { "CpcmdDvld", 11, 1 },
6740         { "CpcmdSave", 10, 1 },
6741         { "RxPstructsFull", 8, 2 },
6742         { "EpcmdDvld", 7, 1 },
6743         { "EpcmdFlush", 6, 1 },
6744         { "EpcmdTrimPrefix", 5, 1 },
6745         { "EpcmdTrimPostfix", 4, 1 },
6746         { "ERssIp4Pkt", 3, 1 },
6747         { "ERssIp6Pkt", 2, 1 },
6748         { "ERssTcpUdpPkt", 1, 1 },
6749         { "ERssFceFipPkt", 0, 1 },
6750         { NULL }
6751 };
6752
6753 static void
6754 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6755 {
6756
6757         field_desc_show(sb, *p, tp_la0);
6758 }
6759
6760 static void
6761 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6762 {
6763
6764         if (idx)
6765                 sbuf_printf(sb, "\n");
6766         field_desc_show(sb, p[0], tp_la0);
6767         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6768                 field_desc_show(sb, p[1], tp_la0);
6769 }
6770
6771 static void
6772 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6773 {
6774
6775         if (idx)
6776                 sbuf_printf(sb, "\n");
6777         field_desc_show(sb, p[0], tp_la0);
6778         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6779                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6780 }
6781
6782 static int
6783 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6784 {
6785         struct adapter *sc = arg1;
6786         struct sbuf *sb;
6787         uint64_t *buf, *p;
6788         int rc;
6789         u_int i, inc;
6790         void (*show_func)(struct sbuf *, uint64_t *, int);
6791
6792         rc = sysctl_wire_old_buffer(req, 0);
6793         if (rc != 0)
6794                 return (rc);
6795
6796         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6797         if (sb == NULL)
6798                 return (ENOMEM);
6799
6800         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6801
6802         t4_tp_read_la(sc, buf, NULL);
6803         p = buf;
6804
6805         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6806         case 2:
6807                 inc = 2;
6808                 show_func = tp_la_show2;
6809                 break;
6810         case 3:
6811                 inc = 2;
6812                 show_func = tp_la_show3;
6813                 break;
6814         default:
6815                 inc = 1;
6816                 show_func = tp_la_show;
6817         }
6818
6819         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6820                 (*show_func)(sb, p, i);
6821
6822         rc = sbuf_finish(sb);
6823         sbuf_delete(sb);
6824         free(buf, M_CXGBE);
6825         return (rc);
6826 }
6827
6828 static int
6829 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6830 {
6831         struct adapter *sc = arg1;
6832         struct sbuf *sb;
6833         int rc;
6834         u64 nrate[NCHAN], orate[NCHAN];
6835
6836         rc = sysctl_wire_old_buffer(req, 0);
6837         if (rc != 0)
6838                 return (rc);
6839
6840         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6841         if (sb == NULL)
6842                 return (ENOMEM);
6843
6844         t4_get_chan_txrate(sc, nrate, orate);
6845         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6846                  "channel 3\n");
6847         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6848             nrate[0], nrate[1], nrate[2], nrate[3]);
6849         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6850             orate[0], orate[1], orate[2], orate[3]);
6851
6852         rc = sbuf_finish(sb);
6853         sbuf_delete(sb);
6854
6855         return (rc);
6856 }
6857
6858 static int
6859 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6860 {
6861         struct adapter *sc = arg1;
6862         struct sbuf *sb;
6863         uint32_t *buf, *p;
6864         int rc, i;
6865
6866         rc = sysctl_wire_old_buffer(req, 0);
6867         if (rc != 0)
6868                 return (rc);
6869
6870         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6871         if (sb == NULL)
6872                 return (ENOMEM);
6873
6874         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6875             M_ZERO | M_WAITOK);
6876
6877         t4_ulprx_read_la(sc, buf);
6878         p = buf;
6879
6880         sbuf_printf(sb, "      Pcmd        Type   Message"
6881             "                Data");
6882         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6883                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6884                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6885         }
6886
6887         rc = sbuf_finish(sb);
6888         sbuf_delete(sb);
6889         free(buf, M_CXGBE);
6890         return (rc);
6891 }
6892
6893 static int
6894 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6895 {
6896         struct adapter *sc = arg1;
6897         struct sbuf *sb;
6898         int rc, v;
6899
6900         rc = sysctl_wire_old_buffer(req, 0);
6901         if (rc != 0)
6902                 return (rc);
6903
6904         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6905         if (sb == NULL)
6906                 return (ENOMEM);
6907
6908         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6909         if (G_STATSOURCE_T5(v) == 7) {
6910                 if (G_STATMODE(v) == 0) {
6911                         sbuf_printf(sb, "total %d, incomplete %d",
6912                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6913                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6914                 } else if (G_STATMODE(v) == 1) {
6915                         sbuf_printf(sb, "total %d, data overflow %d",
6916                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6917                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6918                 }
6919         }
6920         rc = sbuf_finish(sb);
6921         sbuf_delete(sb);
6922
6923         return (rc);
6924 }
6925 #endif
6926
6927 static uint32_t
6928 fconf_to_mode(uint32_t fconf)
6929 {
6930         uint32_t mode;
6931
6932         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6933             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6934
6935         if (fconf & F_FRAGMENTATION)
6936                 mode |= T4_FILTER_IP_FRAGMENT;
6937
6938         if (fconf & F_MPSHITTYPE)
6939                 mode |= T4_FILTER_MPS_HIT_TYPE;
6940
6941         if (fconf & F_MACMATCH)
6942                 mode |= T4_FILTER_MAC_IDX;
6943
6944         if (fconf & F_ETHERTYPE)
6945                 mode |= T4_FILTER_ETH_TYPE;
6946
6947         if (fconf & F_PROTOCOL)
6948                 mode |= T4_FILTER_IP_PROTO;
6949
6950         if (fconf & F_TOS)
6951                 mode |= T4_FILTER_IP_TOS;
6952
6953         if (fconf & F_VLAN)
6954                 mode |= T4_FILTER_VLAN;
6955
6956         if (fconf & F_VNIC_ID)
6957                 mode |= T4_FILTER_VNIC;
6958
6959         if (fconf & F_PORT)
6960                 mode |= T4_FILTER_PORT;
6961
6962         if (fconf & F_FCOE)
6963                 mode |= T4_FILTER_FCoE;
6964
6965         return (mode);
6966 }
6967
6968 static uint32_t
6969 mode_to_fconf(uint32_t mode)
6970 {
6971         uint32_t fconf = 0;
6972
6973         if (mode & T4_FILTER_IP_FRAGMENT)
6974                 fconf |= F_FRAGMENTATION;
6975
6976         if (mode & T4_FILTER_MPS_HIT_TYPE)
6977                 fconf |= F_MPSHITTYPE;
6978
6979         if (mode & T4_FILTER_MAC_IDX)
6980                 fconf |= F_MACMATCH;
6981
6982         if (mode & T4_FILTER_ETH_TYPE)
6983                 fconf |= F_ETHERTYPE;
6984
6985         if (mode & T4_FILTER_IP_PROTO)
6986                 fconf |= F_PROTOCOL;
6987
6988         if (mode & T4_FILTER_IP_TOS)
6989                 fconf |= F_TOS;
6990
6991         if (mode & T4_FILTER_VLAN)
6992                 fconf |= F_VLAN;
6993
6994         if (mode & T4_FILTER_VNIC)
6995                 fconf |= F_VNIC_ID;
6996
6997         if (mode & T4_FILTER_PORT)
6998                 fconf |= F_PORT;
6999
7000         if (mode & T4_FILTER_FCoE)
7001                 fconf |= F_FCOE;
7002
7003         return (fconf);
7004 }
7005
7006 static uint32_t
7007 fspec_to_fconf(struct t4_filter_specification *fs)
7008 {
7009         uint32_t fconf = 0;
7010
7011         if (fs->val.frag || fs->mask.frag)
7012                 fconf |= F_FRAGMENTATION;
7013
7014         if (fs->val.matchtype || fs->mask.matchtype)
7015                 fconf |= F_MPSHITTYPE;
7016
7017         if (fs->val.macidx || fs->mask.macidx)
7018                 fconf |= F_MACMATCH;
7019
7020         if (fs->val.ethtype || fs->mask.ethtype)
7021                 fconf |= F_ETHERTYPE;
7022
7023         if (fs->val.proto || fs->mask.proto)
7024                 fconf |= F_PROTOCOL;
7025
7026         if (fs->val.tos || fs->mask.tos)
7027                 fconf |= F_TOS;
7028
7029         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7030                 fconf |= F_VLAN;
7031
7032         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7033                 fconf |= F_VNIC_ID;
7034
7035         if (fs->val.iport || fs->mask.iport)
7036                 fconf |= F_PORT;
7037
7038         if (fs->val.fcoe || fs->mask.fcoe)
7039                 fconf |= F_FCOE;
7040
7041         return (fconf);
7042 }
7043
7044 static int
7045 get_filter_mode(struct adapter *sc, uint32_t *mode)
7046 {
7047         int rc;
7048         uint32_t fconf;
7049
7050         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7051             "t4getfm");
7052         if (rc)
7053                 return (rc);
7054
7055         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7056             A_TP_VLAN_PRI_MAP);
7057
7058         if (sc->params.tp.vlan_pri_map != fconf) {
7059                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7060                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7061                     fconf);
7062         }
7063
7064         *mode = fconf_to_mode(fconf);
7065
7066         end_synchronized_op(sc, LOCK_HELD);
7067         return (0);
7068 }
7069
7070 static int
7071 set_filter_mode(struct adapter *sc, uint32_t mode)
7072 {
7073         uint32_t fconf;
7074         int rc;
7075
7076         fconf = mode_to_fconf(mode);
7077
7078         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7079             "t4setfm");
7080         if (rc)
7081                 return (rc);
7082
7083         if (sc->tids.ftids_in_use > 0) {
7084                 rc = EBUSY;
7085                 goto done;
7086         }
7087
7088 #ifdef TCP_OFFLOAD
7089         if (uld_active(sc, ULD_TOM)) {
7090                 rc = EBUSY;
7091                 goto done;
7092         }
7093 #endif
7094
7095         rc = -t4_set_filter_mode(sc, fconf);
7096 done:
7097         end_synchronized_op(sc, LOCK_HELD);
7098         return (rc);
7099 }
7100
7101 static inline uint64_t
7102 get_filter_hits(struct adapter *sc, uint32_t fid)
7103 {
7104         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7105         uint64_t hits;
7106
7107         memwin_info(sc, 0, &mw_base, NULL);
7108         off = position_memwin(sc, 0,
7109             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7110         if (is_t4(sc)) {
7111                 hits = t4_read_reg64(sc, mw_base + off + 16);
7112                 hits = be64toh(hits);
7113         } else {
7114                 hits = t4_read_reg(sc, mw_base + off + 24);
7115                 hits = be32toh(hits);
7116         }
7117
7118         return (hits);
7119 }
7120
7121 static int
7122 get_filter(struct adapter *sc, struct t4_filter *t)
7123 {
7124         int i, rc, nfilters = sc->tids.nftids;
7125         struct filter_entry *f;
7126
7127         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7128             "t4getf");
7129         if (rc)
7130                 return (rc);
7131
7132         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7133             t->idx >= nfilters) {
7134                 t->idx = 0xffffffff;
7135                 goto done;
7136         }
7137
7138         f = &sc->tids.ftid_tab[t->idx];
7139         for (i = t->idx; i < nfilters; i++, f++) {
7140                 if (f->valid) {
7141                         t->idx = i;
7142                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7143                         t->smtidx = f->smtidx;
7144                         if (f->fs.hitcnts)
7145                                 t->hits = get_filter_hits(sc, t->idx);
7146                         else
7147                                 t->hits = UINT64_MAX;
7148                         t->fs = f->fs;
7149
7150                         goto done;
7151                 }
7152         }
7153
7154         t->idx = 0xffffffff;
7155 done:
7156         end_synchronized_op(sc, LOCK_HELD);
7157         return (0);
7158 }
7159
7160 static int
7161 set_filter(struct adapter *sc, struct t4_filter *t)
7162 {
7163         unsigned int nfilters, nports;
7164         struct filter_entry *f;
7165         int i, rc;
7166
7167         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7168         if (rc)
7169                 return (rc);
7170
7171         nfilters = sc->tids.nftids;
7172         nports = sc->params.nports;
7173
7174         if (nfilters == 0) {
7175                 rc = ENOTSUP;
7176                 goto done;
7177         }
7178
7179         if (!(sc->flags & FULL_INIT_DONE)) {
7180                 rc = EAGAIN;
7181                 goto done;
7182         }
7183
7184         if (t->idx >= nfilters) {
7185                 rc = EINVAL;
7186                 goto done;
7187         }
7188
7189         /* Validate against the global filter mode */
7190         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7191             sc->params.tp.vlan_pri_map) {
7192                 rc = E2BIG;
7193                 goto done;
7194         }
7195
7196         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7197                 rc = EINVAL;
7198                 goto done;
7199         }
7200
7201         if (t->fs.val.iport >= nports) {
7202                 rc = EINVAL;
7203                 goto done;
7204         }
7205
7206         /* Can't specify an iq if not steering to it */
7207         if (!t->fs.dirsteer && t->fs.iq) {
7208                 rc = EINVAL;
7209                 goto done;
7210         }
7211
7212         /* IPv6 filter idx must be 4 aligned */
7213         if (t->fs.type == 1 &&
7214             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7215                 rc = EINVAL;
7216                 goto done;
7217         }
7218
7219         if (sc->tids.ftid_tab == NULL) {
7220                 KASSERT(sc->tids.ftids_in_use == 0,
7221                     ("%s: no memory allocated but filters_in_use > 0",
7222                     __func__));
7223
7224                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7225                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7226                 if (sc->tids.ftid_tab == NULL) {
7227                         rc = ENOMEM;
7228                         goto done;
7229                 }
7230                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7231         }
7232
7233         for (i = 0; i < 4; i++) {
7234                 f = &sc->tids.ftid_tab[t->idx + i];
7235
7236                 if (f->pending || f->valid) {
7237                         rc = EBUSY;
7238                         goto done;
7239                 }
7240                 if (f->locked) {
7241                         rc = EPERM;
7242                         goto done;
7243                 }
7244
7245                 if (t->fs.type == 0)
7246                         break;
7247         }
7248
7249         f = &sc->tids.ftid_tab[t->idx];
7250         f->fs = t->fs;
7251
7252         rc = set_filter_wr(sc, t->idx);
7253 done:
7254         end_synchronized_op(sc, 0);
7255
7256         if (rc == 0) {
7257                 mtx_lock(&sc->tids.ftid_lock);
7258                 for (;;) {
7259                         if (f->pending == 0) {
7260                                 rc = f->valid ? 0 : EIO;
7261                                 break;
7262                         }
7263
7264                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7265                             PCATCH, "t4setfw", 0)) {
7266                                 rc = EINPROGRESS;
7267                                 break;
7268                         }
7269                 }
7270                 mtx_unlock(&sc->tids.ftid_lock);
7271         }
7272         return (rc);
7273 }
7274
7275 static int
7276 del_filter(struct adapter *sc, struct t4_filter *t)
7277 {
7278         unsigned int nfilters;
7279         struct filter_entry *f;
7280         int rc;
7281
7282         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7283         if (rc)
7284                 return (rc);
7285
7286         nfilters = sc->tids.nftids;
7287
7288         if (nfilters == 0) {
7289                 rc = ENOTSUP;
7290                 goto done;
7291         }
7292
7293         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7294             t->idx >= nfilters) {
7295                 rc = EINVAL;
7296                 goto done;
7297         }
7298
7299         if (!(sc->flags & FULL_INIT_DONE)) {
7300                 rc = EAGAIN;
7301                 goto done;
7302         }
7303
7304         f = &sc->tids.ftid_tab[t->idx];
7305
7306         if (f->pending) {
7307                 rc = EBUSY;
7308                 goto done;
7309         }
7310         if (f->locked) {
7311                 rc = EPERM;
7312                 goto done;
7313         }
7314
7315         if (f->valid) {
7316                 t->fs = f->fs;  /* extra info for the caller */
7317                 rc = del_filter_wr(sc, t->idx);
7318         }
7319
7320 done:
7321         end_synchronized_op(sc, 0);
7322
7323         if (rc == 0) {
7324                 mtx_lock(&sc->tids.ftid_lock);
7325                 for (;;) {
7326                         if (f->pending == 0) {
7327                                 rc = f->valid ? EIO : 0;
7328                                 break;
7329                         }
7330
7331                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7332                             PCATCH, "t4delfw", 0)) {
7333                                 rc = EINPROGRESS;
7334                                 break;
7335                         }
7336                 }
7337                 mtx_unlock(&sc->tids.ftid_lock);
7338         }
7339
7340         return (rc);
7341 }
7342
7343 static void
7344 clear_filter(struct filter_entry *f)
7345 {
7346         if (f->l2t)
7347                 t4_l2t_release(f->l2t);
7348
7349         bzero(f, sizeof (*f));
7350 }
7351
7352 static int
7353 set_filter_wr(struct adapter *sc, int fidx)
7354 {
7355         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7356         struct fw_filter_wr *fwr;
7357         unsigned int ftid;
7358         struct wrq_cookie cookie;
7359
7360         ASSERT_SYNCHRONIZED_OP(sc);
7361
7362         if (f->fs.newdmac || f->fs.newvlan) {
7363                 /* This filter needs an L2T entry; allocate one. */
7364                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7365                 if (f->l2t == NULL)
7366                         return (EAGAIN);
7367                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7368                     f->fs.dmac)) {
7369                         t4_l2t_release(f->l2t);
7370                         f->l2t = NULL;
7371                         return (ENOMEM);
7372                 }
7373         }
7374
7375         ftid = sc->tids.ftid_base + fidx;
7376
7377         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7378         if (fwr == NULL)
7379                 return (ENOMEM);
7380         bzero(fwr, sizeof(*fwr));
7381
7382         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7383         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7384         fwr->tid_to_iq =
7385             htobe32(V_FW_FILTER_WR_TID(ftid) |
7386                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7387                 V_FW_FILTER_WR_NOREPLY(0) |
7388                 V_FW_FILTER_WR_IQ(f->fs.iq));
7389         fwr->del_filter_to_l2tix =
7390             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7391                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7392                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7393                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7394                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7395                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7396                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7397                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7398                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7399                     f->fs.newvlan == VLAN_REWRITE) |
7400                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7401                     f->fs.newvlan == VLAN_REWRITE) |
7402                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7403                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7404                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7405                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7406         fwr->ethtype = htobe16(f->fs.val.ethtype);
7407         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7408         fwr->frag_to_ovlan_vldm =
7409             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7410                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7411                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7412                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7413                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7414                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7415         fwr->smac_sel = 0;
7416         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7417             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7418         fwr->maci_to_matchtypem =
7419             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7420                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7421                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7422                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7423                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7424                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7425                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7426                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7427         fwr->ptcl = f->fs.val.proto;
7428         fwr->ptclm = f->fs.mask.proto;
7429         fwr->ttyp = f->fs.val.tos;
7430         fwr->ttypm = f->fs.mask.tos;
7431         fwr->ivlan = htobe16(f->fs.val.vlan);
7432         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7433         fwr->ovlan = htobe16(f->fs.val.vnic);
7434         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7435         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7436         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7437         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7438         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7439         fwr->lp = htobe16(f->fs.val.dport);
7440         fwr->lpm = htobe16(f->fs.mask.dport);
7441         fwr->fp = htobe16(f->fs.val.sport);
7442         fwr->fpm = htobe16(f->fs.mask.sport);
7443         if (f->fs.newsmac)
7444                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7445
7446         f->pending = 1;
7447         sc->tids.ftids_in_use++;
7448
7449         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7450         return (0);
7451 }
7452
7453 static int
7454 del_filter_wr(struct adapter *sc, int fidx)
7455 {
7456         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7457         struct fw_filter_wr *fwr;
7458         unsigned int ftid;
7459         struct wrq_cookie cookie;
7460
7461         ftid = sc->tids.ftid_base + fidx;
7462
7463         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7464         if (fwr == NULL)
7465                 return (ENOMEM);
7466         bzero(fwr, sizeof (*fwr));
7467
7468         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7469
7470         f->pending = 1;
7471         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7472         return (0);
7473 }
7474
7475 int
7476 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7477 {
7478         struct adapter *sc = iq->adapter;
7479         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7480         unsigned int idx = GET_TID(rpl);
7481         unsigned int rc;
7482         struct filter_entry *f;
7483
7484         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7485             rss->opcode));
7486
7487         if (is_ftid(sc, idx)) {
7488
7489                 idx -= sc->tids.ftid_base;
7490                 f = &sc->tids.ftid_tab[idx];
7491                 rc = G_COOKIE(rpl->cookie);
7492
7493                 mtx_lock(&sc->tids.ftid_lock);
7494                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7495                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7496                             __func__, idx));
7497                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7498                         f->pending = 0;  /* asynchronous setup completed */
7499                         f->valid = 1;
7500                 } else {
7501                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7502                                 /* Add or delete failed, display an error */
7503                                 log(LOG_ERR,
7504                                     "filter %u setup failed with error %u\n",
7505                                     idx, rc);
7506                         }
7507
7508                         clear_filter(f);
7509                         sc->tids.ftids_in_use--;
7510                 }
7511                 wakeup(&sc->tids.ftid_tab);
7512                 mtx_unlock(&sc->tids.ftid_lock);
7513         }
7514
7515         return (0);
7516 }
7517
7518 static int
7519 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7520 {
7521         int rc;
7522
7523         if (cntxt->cid > M_CTXTQID)
7524                 return (EINVAL);
7525
7526         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7527             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7528                 return (EINVAL);
7529
7530         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7531         if (rc)
7532                 return (rc);
7533
7534         if (sc->flags & FW_OK) {
7535                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7536                     &cntxt->data[0]);
7537                 if (rc == 0)
7538                         goto done;
7539         }
7540
7541         /*
7542          * Read via firmware failed or wasn't even attempted.  Read directly via
7543          * the backdoor.
7544          */
7545         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7546 done:
7547         end_synchronized_op(sc, 0);
7548         return (rc);
7549 }
7550
7551 static int
7552 load_fw(struct adapter *sc, struct t4_data *fw)
7553 {
7554         int rc;
7555         uint8_t *fw_data;
7556
7557         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7558         if (rc)
7559                 return (rc);
7560
7561         if (sc->flags & FULL_INIT_DONE) {
7562                 rc = EBUSY;
7563                 goto done;
7564         }
7565
7566         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7567         if (fw_data == NULL) {
7568                 rc = ENOMEM;
7569                 goto done;
7570         }
7571
7572         rc = copyin(fw->data, fw_data, fw->len);
7573         if (rc == 0)
7574                 rc = -t4_load_fw(sc, fw_data, fw->len);
7575
7576         free(fw_data, M_CXGBE);
7577 done:
7578         end_synchronized_op(sc, 0);
7579         return (rc);
7580 }
7581
7582 static int
7583 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7584 {
7585         uint32_t addr, off, remaining, i, n;
7586         uint32_t *buf, *b;
7587         uint32_t mw_base, mw_aperture;
7588         int rc;
7589         uint8_t *dst;
7590
7591         rc = validate_mem_range(sc, mr->addr, mr->len);
7592         if (rc != 0)
7593                 return (rc);
7594
7595         memwin_info(sc, win, &mw_base, &mw_aperture);
7596         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7597         addr = mr->addr;
7598         remaining = mr->len;
7599         dst = (void *)mr->data;
7600
7601         while (remaining) {
7602                 off = position_memwin(sc, win, addr);
7603
7604                 /* number of bytes that we'll copy in the inner loop */
7605                 n = min(remaining, mw_aperture - off);
7606                 for (i = 0; i < n; i += 4)
7607                         *b++ = t4_read_reg(sc, mw_base + off + i);
7608
7609                 rc = copyout(buf, dst, n);
7610                 if (rc != 0)
7611                         break;
7612
7613                 b = buf;
7614                 dst += n;
7615                 remaining -= n;
7616                 addr += n;
7617         }
7618
7619         free(buf, M_CXGBE);
7620         return (rc);
7621 }
7622
7623 static int
7624 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7625 {
7626         int rc;
7627
7628         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7629                 return (EINVAL);
7630
7631         if (i2cd->len > sizeof(i2cd->data))
7632                 return (EFBIG);
7633
7634         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7635         if (rc)
7636                 return (rc);
7637         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7638             i2cd->offset, i2cd->len, &i2cd->data[0]);
7639         end_synchronized_op(sc, 0);
7640
7641         return (rc);
7642 }
7643
7644 static int
7645 in_range(int val, int lo, int hi)
7646 {
7647
7648         return (val < 0 || (val <= hi && val >= lo));
7649 }
7650
7651 static int
7652 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7653 {
7654         int fw_subcmd, fw_type, rc;
7655
7656         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7657         if (rc)
7658                 return (rc);
7659
7660         if (!(sc->flags & FULL_INIT_DONE)) {
7661                 rc = EAGAIN;
7662                 goto done;
7663         }
7664
7665         /*
7666          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7667          * sub-command and type are in common locations.)
7668          */
7669         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7670                 fw_subcmd = FW_SCHED_SC_CONFIG;
7671         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7672                 fw_subcmd = FW_SCHED_SC_PARAMS;
7673         else {
7674                 rc = EINVAL;
7675                 goto done;
7676         }
7677         if (p->type == SCHED_CLASS_TYPE_PACKET)
7678                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7679         else {
7680                 rc = EINVAL;
7681                 goto done;
7682         }
7683
7684         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7685                 /* Vet our parameters ..*/
7686                 if (p->u.config.minmax < 0) {
7687                         rc = EINVAL;
7688                         goto done;
7689                 }
7690
7691                 /* And pass the request to the firmware ...*/
7692                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7693                 goto done;
7694         }
7695
7696         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7697                 int fw_level;
7698                 int fw_mode;
7699                 int fw_rateunit;
7700                 int fw_ratemode;
7701
7702                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7703                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7704                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7705                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7706                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7707                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7708                 else {
7709                         rc = EINVAL;
7710                         goto done;
7711                 }
7712
7713                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7714                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7715                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7716                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7717                 else {
7718                         rc = EINVAL;
7719                         goto done;
7720                 }
7721
7722                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7723                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7724                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7725                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7726                 else {
7727                         rc = EINVAL;
7728                         goto done;
7729                 }
7730
7731                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7732                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7733                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7734                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7735                 else {
7736                         rc = EINVAL;
7737                         goto done;
7738                 }
7739
7740                 /* Vet our parameters ... */
7741                 if (!in_range(p->u.params.channel, 0, 3) ||
7742                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7743                     !in_range(p->u.params.minrate, 0, 10000000) ||
7744                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7745                     !in_range(p->u.params.weight, 0, 100)) {
7746                         rc = ERANGE;
7747                         goto done;
7748                 }
7749
7750                 /*
7751                  * Translate any unset parameters into the firmware's
7752                  * nomenclature and/or fail the call if the parameters
7753                  * are required ...
7754                  */
7755                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7756                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7757                         rc = EINVAL;
7758                         goto done;
7759                 }
7760                 if (p->u.params.minrate < 0)
7761                         p->u.params.minrate = 0;
7762                 if (p->u.params.maxrate < 0) {
7763                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7764                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7765                                 rc = EINVAL;
7766                                 goto done;
7767                         } else
7768                                 p->u.params.maxrate = 0;
7769                 }
7770                 if (p->u.params.weight < 0) {
7771                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7772                                 rc = EINVAL;
7773                                 goto done;
7774                         } else
7775                                 p->u.params.weight = 0;
7776                 }
7777                 if (p->u.params.pktsize < 0) {
7778                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7779                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7780                                 rc = EINVAL;
7781                                 goto done;
7782                         } else
7783                                 p->u.params.pktsize = 0;
7784                 }
7785
7786                 /* See what the firmware thinks of the request ... */
7787                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7788                     fw_rateunit, fw_ratemode, p->u.params.channel,
7789                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7790                     p->u.params.weight, p->u.params.pktsize, 1);
7791                 goto done;
7792         }
7793
7794         rc = EINVAL;
7795 done:
7796         end_synchronized_op(sc, 0);
7797         return (rc);
7798 }
7799
7800 static int
7801 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7802 {
7803         struct port_info *pi = NULL;
7804         struct sge_txq *txq;
7805         uint32_t fw_mnem, fw_queue, fw_class;
7806         int i, rc;
7807
7808         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7809         if (rc)
7810                 return (rc);
7811
7812         if (!(sc->flags & FULL_INIT_DONE)) {
7813                 rc = EAGAIN;
7814                 goto done;
7815         }
7816
7817         if (p->port >= sc->params.nports) {
7818                 rc = EINVAL;
7819                 goto done;
7820         }
7821
7822         pi = sc->port[p->port];
7823         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7824                 rc = EINVAL;
7825                 goto done;
7826         }
7827
7828         /*
7829          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7830          * Scheduling Class in this case).
7831          */
7832         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7833             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7834         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7835
7836         /*
7837          * If op.queue is non-negative, then we're only changing the scheduling
7838          * on a single specified TX queue.
7839          */
7840         if (p->queue >= 0) {
7841                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7842                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7843                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7844                     &fw_class);
7845                 goto done;
7846         }
7847
7848         /*
7849          * Change the scheduling on all the TX queues for the
7850          * interface.
7851          */
7852         for_each_txq(pi, i, txq) {
7853                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7854                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7855                     &fw_class);
7856                 if (rc)
7857                         goto done;
7858         }
7859
7860         rc = 0;
7861 done:
7862         end_synchronized_op(sc, 0);
7863         return (rc);
7864 }
7865
7866 int
7867 t4_os_find_pci_capability(struct adapter *sc, int cap)
7868 {
7869         int i;
7870
7871         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7872 }
7873
7874 int
7875 t4_os_pci_save_state(struct adapter *sc)
7876 {
7877         device_t dev;
7878         struct pci_devinfo *dinfo;
7879
7880         dev = sc->dev;
7881         dinfo = device_get_ivars(dev);
7882
7883         pci_cfg_save(dev, dinfo, 0);
7884         return (0);
7885 }
7886
7887 int
7888 t4_os_pci_restore_state(struct adapter *sc)
7889 {
7890         device_t dev;
7891         struct pci_devinfo *dinfo;
7892
7893         dev = sc->dev;
7894         dinfo = device_get_ivars(dev);
7895
7896         pci_cfg_restore(dev, dinfo);
7897         return (0);
7898 }
7899
7900 void
7901 t4_os_portmod_changed(const struct adapter *sc, int idx)
7902 {
7903         struct port_info *pi = sc->port[idx];
7904         static const char *mod_str[] = {
7905                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7906         };
7907
7908         build_medialist(pi, &pi->media);
7909 #ifdef DEV_NETMAP
7910         build_medialist(pi, &pi->nm_media);
7911 #endif
7912
7913         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7914                 if_printf(pi->ifp, "transceiver unplugged.\n");
7915         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7916                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7917         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7918                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7919         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7920                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7921                     mod_str[pi->mod_type]);
7922         } else {
7923                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7924                     pi->mod_type);
7925         }
7926 }
7927
7928 void
7929 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7930 {
7931         struct port_info *pi = sc->port[idx];
7932         struct ifnet *ifp = pi->ifp;
7933
7934         if (link_stat) {
7935                 pi->linkdnrc = -1;
7936                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7937                 if_link_state_change(ifp, LINK_STATE_UP);
7938         } else {
7939                 if (reason >= 0)
7940                         pi->linkdnrc = reason;
7941                 if_link_state_change(ifp, LINK_STATE_DOWN);
7942         }
7943 }
7944
7945 void
7946 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7947 {
7948         struct adapter *sc;
7949
7950         sx_slock(&t4_list_lock);
7951         SLIST_FOREACH(sc, &t4_list, link) {
7952                 /*
7953                  * func should not make any assumptions about what state sc is
7954                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7955                  */
7956                 func(sc, arg);
7957         }
7958         sx_sunlock(&t4_list_lock);
7959 }
7960
7961 static int
7962 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7963 {
7964        return (0);
7965 }
7966
7967 static int
7968 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7969 {
7970        return (0);
7971 }
7972
7973 static int
7974 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7975     struct thread *td)
7976 {
7977         int rc;
7978         struct adapter *sc = dev->si_drv1;
7979
7980         rc = priv_check(td, PRIV_DRIVER);
7981         if (rc != 0)
7982                 return (rc);
7983
7984         switch (cmd) {
7985         case CHELSIO_T4_GETREG: {
7986                 struct t4_reg *edata = (struct t4_reg *)data;
7987
7988                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7989                         return (EFAULT);
7990
7991                 if (edata->size == 4)
7992                         edata->val = t4_read_reg(sc, edata->addr);
7993                 else if (edata->size == 8)
7994                         edata->val = t4_read_reg64(sc, edata->addr);
7995                 else
7996                         return (EINVAL);
7997
7998                 break;
7999         }
8000         case CHELSIO_T4_SETREG: {
8001                 struct t4_reg *edata = (struct t4_reg *)data;
8002
8003                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8004                         return (EFAULT);
8005
8006                 if (edata->size == 4) {
8007                         if (edata->val & 0xffffffff00000000)
8008                                 return (EINVAL);
8009                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8010                 } else if (edata->size == 8)
8011                         t4_write_reg64(sc, edata->addr, edata->val);
8012                 else
8013                         return (EINVAL);
8014                 break;
8015         }
8016         case CHELSIO_T4_REGDUMP: {
8017                 struct t4_regdump *regs = (struct t4_regdump *)data;
8018                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8019                 uint8_t *buf;
8020
8021                 if (regs->len < reglen) {
8022                         regs->len = reglen; /* hint to the caller */
8023                         return (ENOBUFS);
8024                 }
8025
8026                 regs->len = reglen;
8027                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8028                 t4_get_regs(sc, regs, buf);
8029                 rc = copyout(buf, regs->data, reglen);
8030                 free(buf, M_CXGBE);
8031                 break;
8032         }
8033         case CHELSIO_T4_GET_FILTER_MODE:
8034                 rc = get_filter_mode(sc, (uint32_t *)data);
8035                 break;
8036         case CHELSIO_T4_SET_FILTER_MODE:
8037                 rc = set_filter_mode(sc, *(uint32_t *)data);
8038                 break;
8039         case CHELSIO_T4_GET_FILTER:
8040                 rc = get_filter(sc, (struct t4_filter *)data);
8041                 break;
8042         case CHELSIO_T4_SET_FILTER:
8043                 rc = set_filter(sc, (struct t4_filter *)data);
8044                 break;
8045         case CHELSIO_T4_DEL_FILTER:
8046                 rc = del_filter(sc, (struct t4_filter *)data);
8047                 break;
8048         case CHELSIO_T4_GET_SGE_CONTEXT:
8049                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8050                 break;
8051         case CHELSIO_T4_LOAD_FW:
8052                 rc = load_fw(sc, (struct t4_data *)data);
8053                 break;
8054         case CHELSIO_T4_GET_MEM:
8055                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8056                 break;
8057         case CHELSIO_T4_GET_I2C:
8058                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8059                 break;
8060         case CHELSIO_T4_CLEAR_STATS: {
8061                 int i;
8062                 u_int port_id = *(uint32_t *)data;
8063                 struct port_info *pi;
8064
8065                 if (port_id >= sc->params.nports)
8066                         return (EINVAL);
8067                 pi = sc->port[port_id];
8068
8069                 /* MAC stats */
8070                 t4_clr_port_stats(sc, pi->tx_chan);
8071                 pi->tx_parse_error = 0;
8072
8073                 if (pi->flags & PORT_INIT_DONE) {
8074                         struct sge_rxq *rxq;
8075                         struct sge_txq *txq;
8076                         struct sge_wrq *wrq;
8077
8078                         for_each_rxq(pi, i, rxq) {
8079 #if defined(INET) || defined(INET6)
8080                                 rxq->lro.lro_queued = 0;
8081                                 rxq->lro.lro_flushed = 0;
8082 #endif
8083                                 rxq->rxcsum = 0;
8084                                 rxq->vlan_extraction = 0;
8085                         }
8086
8087                         for_each_txq(pi, i, txq) {
8088                                 txq->txcsum = 0;
8089                                 txq->tso_wrs = 0;
8090                                 txq->vlan_insertion = 0;
8091                                 txq->imm_wrs = 0;
8092                                 txq->sgl_wrs = 0;
8093                                 txq->txpkt_wrs = 0;
8094                                 txq->txpkts0_wrs = 0;
8095                                 txq->txpkts1_wrs = 0;
8096                                 txq->txpkts0_pkts = 0;
8097                                 txq->txpkts1_pkts = 0;
8098                                 mp_ring_reset_stats(txq->r);
8099                         }
8100
8101 #ifdef TCP_OFFLOAD
8102                         /* nothing to clear for each ofld_rxq */
8103
8104                         for_each_ofld_txq(pi, i, wrq) {
8105                                 wrq->tx_wrs_direct = 0;
8106                                 wrq->tx_wrs_copied = 0;
8107                         }
8108 #endif
8109                         wrq = &sc->sge.ctrlq[pi->port_id];
8110                         wrq->tx_wrs_direct = 0;
8111                         wrq->tx_wrs_copied = 0;
8112                 }
8113                 break;
8114         }
8115         case CHELSIO_T4_SCHED_CLASS:
8116                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8117                 break;
8118         case CHELSIO_T4_SCHED_QUEUE:
8119                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8120                 break;
8121         case CHELSIO_T4_GET_TRACER:
8122                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8123                 break;
8124         case CHELSIO_T4_SET_TRACER:
8125                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8126                 break;
8127         default:
8128                 rc = EINVAL;
8129         }
8130
8131         return (rc);
8132 }
8133
8134 #ifdef TCP_OFFLOAD
8135 void
8136 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8137     const unsigned int *pgsz_order)
8138 {
8139         struct port_info *pi = ifp->if_softc;
8140         struct adapter *sc = pi->adapter;
8141
8142         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8143         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8144                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8145                 V_HPZ3(pgsz_order[3]));
8146 }
8147
8148 static int
8149 toe_capability(struct port_info *pi, int enable)
8150 {
8151         int rc;
8152         struct adapter *sc = pi->adapter;
8153
8154         ASSERT_SYNCHRONIZED_OP(sc);
8155
8156         if (!is_offload(sc))
8157                 return (ENODEV);
8158
8159         if (enable) {
8160                 /*
8161                  * We need the port's queues around so that we're able to send
8162                  * and receive CPLs to/from the TOE even if the ifnet for this
8163                  * port has never been UP'd administratively.
8164                  */
8165                 if (!(pi->flags & PORT_INIT_DONE)) {
8166                         rc = cxgbe_init_synchronized(pi);
8167                         if (rc)
8168                                 return (rc);
8169                 }
8170
8171                 if (isset(&sc->offload_map, pi->port_id))
8172                         return (0);
8173
8174                 if (!uld_active(sc, ULD_TOM)) {
8175                         rc = t4_activate_uld(sc, ULD_TOM);
8176                         if (rc == EAGAIN) {
8177                                 log(LOG_WARNING,
8178                                     "You must kldload t4_tom.ko before trying "
8179                                     "to enable TOE on a cxgbe interface.\n");
8180                         }
8181                         if (rc != 0)
8182                                 return (rc);
8183                         KASSERT(sc->tom_softc != NULL,
8184                             ("%s: TOM activated but softc NULL", __func__));
8185                         KASSERT(uld_active(sc, ULD_TOM),
8186                             ("%s: TOM activated but flag not set", __func__));
8187                 }
8188
8189                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8190                 if (!uld_active(sc, ULD_IWARP))
8191                         (void) t4_activate_uld(sc, ULD_IWARP);
8192                 if (!uld_active(sc, ULD_ISCSI))
8193                         (void) t4_activate_uld(sc, ULD_ISCSI);
8194
8195                 setbit(&sc->offload_map, pi->port_id);
8196         } else {
8197                 if (!isset(&sc->offload_map, pi->port_id))
8198                         return (0);
8199
8200                 KASSERT(uld_active(sc, ULD_TOM),
8201                     ("%s: TOM never initialized?", __func__));
8202                 clrbit(&sc->offload_map, pi->port_id);
8203         }
8204
8205         return (0);
8206 }
8207
8208 /*
8209  * Add an upper layer driver to the global list.
8210  */
8211 int
8212 t4_register_uld(struct uld_info *ui)
8213 {
8214         int rc = 0;
8215         struct uld_info *u;
8216
8217         sx_xlock(&t4_uld_list_lock);
8218         SLIST_FOREACH(u, &t4_uld_list, link) {
8219             if (u->uld_id == ui->uld_id) {
8220                     rc = EEXIST;
8221                     goto done;
8222             }
8223         }
8224
8225         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8226         ui->refcount = 0;
8227 done:
8228         sx_xunlock(&t4_uld_list_lock);
8229         return (rc);
8230 }
8231
8232 int
8233 t4_unregister_uld(struct uld_info *ui)
8234 {
8235         int rc = EINVAL;
8236         struct uld_info *u;
8237
8238         sx_xlock(&t4_uld_list_lock);
8239
8240         SLIST_FOREACH(u, &t4_uld_list, link) {
8241             if (u == ui) {
8242                     if (ui->refcount > 0) {
8243                             rc = EBUSY;
8244                             goto done;
8245                     }
8246
8247                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8248                     rc = 0;
8249                     goto done;
8250             }
8251         }
8252 done:
8253         sx_xunlock(&t4_uld_list_lock);
8254         return (rc);
8255 }
8256
8257 int
8258 t4_activate_uld(struct adapter *sc, int id)
8259 {
8260         int rc;
8261         struct uld_info *ui;
8262
8263         ASSERT_SYNCHRONIZED_OP(sc);
8264
8265         if (id < 0 || id > ULD_MAX)
8266                 return (EINVAL);
8267         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8268
8269         sx_slock(&t4_uld_list_lock);
8270
8271         SLIST_FOREACH(ui, &t4_uld_list, link) {
8272                 if (ui->uld_id == id) {
8273                         if (!(sc->flags & FULL_INIT_DONE)) {
8274                                 rc = adapter_full_init(sc);
8275                                 if (rc != 0)
8276                                         break;
8277                         }
8278
8279                         rc = ui->activate(sc);
8280                         if (rc == 0) {
8281                                 setbit(&sc->active_ulds, id);
8282                                 ui->refcount++;
8283                         }
8284                         break;
8285                 }
8286         }
8287
8288         sx_sunlock(&t4_uld_list_lock);
8289
8290         return (rc);
8291 }
8292
8293 int
8294 t4_deactivate_uld(struct adapter *sc, int id)
8295 {
8296         int rc;
8297         struct uld_info *ui;
8298
8299         ASSERT_SYNCHRONIZED_OP(sc);
8300
8301         if (id < 0 || id > ULD_MAX)
8302                 return (EINVAL);
8303         rc = ENXIO;
8304
8305         sx_slock(&t4_uld_list_lock);
8306
8307         SLIST_FOREACH(ui, &t4_uld_list, link) {
8308                 if (ui->uld_id == id) {
8309                         rc = ui->deactivate(sc);
8310                         if (rc == 0) {
8311                                 clrbit(&sc->active_ulds, id);
8312                                 ui->refcount--;
8313                         }
8314                         break;
8315                 }
8316         }
8317
8318         sx_sunlock(&t4_uld_list_lock);
8319
8320         return (rc);
8321 }
8322
8323 int
8324 uld_active(struct adapter *sc, int uld_id)
8325 {
8326
8327         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8328
8329         return (isset(&sc->active_ulds, uld_id));
8330 }
8331 #endif
8332
8333 /*
8334  * Come up with reasonable defaults for some of the tunables, provided they're
8335  * not set by the user (in which case we'll use the values as is).
8336  */
8337 static void
8338 tweak_tunables(void)
8339 {
8340         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8341
8342         if (t4_ntxq10g < 1)
8343                 t4_ntxq10g = min(nc, NTXQ_10G);
8344
8345         if (t4_ntxq1g < 1)
8346                 t4_ntxq1g = min(nc, NTXQ_1G);
8347
8348         if (t4_nrxq10g < 1)
8349                 t4_nrxq10g = min(nc, NRXQ_10G);
8350
8351         if (t4_nrxq1g < 1)
8352                 t4_nrxq1g = min(nc, NRXQ_1G);
8353
8354 #ifdef TCP_OFFLOAD
8355         if (t4_nofldtxq10g < 1)
8356                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8357
8358         if (t4_nofldtxq1g < 1)
8359                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8360
8361         if (t4_nofldrxq10g < 1)
8362                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8363
8364         if (t4_nofldrxq1g < 1)
8365                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8366
8367         if (t4_toecaps_allowed == -1)
8368                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8369 #else
8370         if (t4_toecaps_allowed == -1)
8371                 t4_toecaps_allowed = 0;
8372 #endif
8373
8374 #ifdef DEV_NETMAP
8375         if (t4_nnmtxq10g < 1)
8376                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8377
8378         if (t4_nnmtxq1g < 1)
8379                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8380
8381         if (t4_nnmrxq10g < 1)
8382                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8383
8384         if (t4_nnmrxq1g < 1)
8385                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8386 #endif
8387
8388         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8389                 t4_tmr_idx_10g = TMR_IDX_10G;
8390
8391         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8392                 t4_pktc_idx_10g = PKTC_IDX_10G;
8393
8394         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8395                 t4_tmr_idx_1g = TMR_IDX_1G;
8396
8397         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8398                 t4_pktc_idx_1g = PKTC_IDX_1G;
8399
8400         if (t4_qsize_txq < 128)
8401                 t4_qsize_txq = 128;
8402
8403         if (t4_qsize_rxq < 128)
8404                 t4_qsize_rxq = 128;
8405         while (t4_qsize_rxq & 7)
8406                 t4_qsize_rxq++;
8407
8408         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8409 }
8410
8411 static struct sx mlu;   /* mod load unload */
8412 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8413
8414 static int
8415 mod_event(module_t mod, int cmd, void *arg)
8416 {
8417         int rc = 0;
8418         static int loaded = 0;
8419
8420         switch (cmd) {
8421         case MOD_LOAD:
8422                 sx_xlock(&mlu);
8423                 if (loaded++ == 0) {
8424                         t4_sge_modload();
8425                         sx_init(&t4_list_lock, "T4/T5 adapters");
8426                         SLIST_INIT(&t4_list);
8427 #ifdef TCP_OFFLOAD
8428                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8429                         SLIST_INIT(&t4_uld_list);
8430 #endif
8431                         t4_tracer_modload();
8432                         tweak_tunables();
8433                 }
8434                 sx_xunlock(&mlu);
8435                 break;
8436
8437         case MOD_UNLOAD:
8438                 sx_xlock(&mlu);
8439                 if (--loaded == 0) {
8440                         int tries;
8441
8442                         sx_slock(&t4_list_lock);
8443                         if (!SLIST_EMPTY(&t4_list)) {
8444                                 rc = EBUSY;
8445                                 sx_sunlock(&t4_list_lock);
8446                                 goto done_unload;
8447                         }
8448 #ifdef TCP_OFFLOAD
8449                         sx_slock(&t4_uld_list_lock);
8450                         if (!SLIST_EMPTY(&t4_uld_list)) {
8451                                 rc = EBUSY;
8452                                 sx_sunlock(&t4_uld_list_lock);
8453                                 sx_sunlock(&t4_list_lock);
8454                                 goto done_unload;
8455                         }
8456 #endif
8457                         tries = 0;
8458                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8459                                 uprintf("%ju clusters with custom free routine "
8460                                     "still is use.\n", t4_sge_extfree_refs());
8461                                 pause("t4unload", 2 * hz);
8462                         }
8463 #ifdef TCP_OFFLOAD
8464                         sx_sunlock(&t4_uld_list_lock);
8465 #endif
8466                         sx_sunlock(&t4_list_lock);
8467
8468                         if (t4_sge_extfree_refs() == 0) {
8469                                 t4_tracer_modunload();
8470 #ifdef TCP_OFFLOAD
8471                                 sx_destroy(&t4_uld_list_lock);
8472 #endif
8473                                 sx_destroy(&t4_list_lock);
8474                                 t4_sge_modunload();
8475                                 loaded = 0;
8476                         } else {
8477                                 rc = EBUSY;
8478                                 loaded++;       /* undo earlier decrement */
8479                         }
8480                 }
8481 done_unload:
8482                 sx_xunlock(&mlu);
8483                 break;
8484         }
8485
8486         return (rc);
8487 }
8488
8489 static devclass_t t4_devclass, t5_devclass;
8490 static devclass_t cxgbe_devclass, cxl_devclass;
8491
8492 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8493 MODULE_VERSION(t4nex, 1);
8494 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8495
8496 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8497 MODULE_VERSION(t5nex, 1);
8498 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8499
8500 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8501 MODULE_VERSION(cxgbe, 1);
8502
8503 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8504 MODULE_VERSION(cxl, 1);