sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69 #include "t4_mp_ring.h"
  70
  71 /* T4 bus driver interface */
  72 static int t4_probe(device_t);
  73 static int t4_attach(device_t);
  74 static int t4_detach(device_t);
  75 static device_method_t t4_methods[] = {
  76         DEVMETHOD(device_probe,         t4_probe),
  77         DEVMETHOD(device_attach,        t4_attach),
  78         DEVMETHOD(device_detach,        t4_detach),
  79
  80         DEVMETHOD_END
  81 };
  82 static driver_t t4_driver = {
  83         "t4nex",
  84         t4_methods,
  85         sizeof(struct adapter)
  86 };
  87
  88
  89 /* T4 port (cxgbe) interface */
  90 static int cxgbe_probe(device_t);
  91 static int cxgbe_attach(device_t);
  92 static int cxgbe_detach(device_t);
  93 static device_method_t cxgbe_methods[] = {
  94         DEVMETHOD(device_probe,         cxgbe_probe),
  95         DEVMETHOD(device_attach,        cxgbe_attach),
  96         DEVMETHOD(device_detach,        cxgbe_detach),
  97         { 0, 0 }
  98 };
  99 static driver_t cxgbe_driver = {
 100         "cxgbe",
 101         cxgbe_methods,
 102         sizeof(struct port_info)
 103 };
 104
 105 static d_ioctl_t t4_ioctl;
 106 static d_open_t t4_open;
 107 static d_close_t t4_close;
 108
 109 static struct cdevsw t4_cdevsw = {
 110        .d_version = D_VERSION,
 111        .d_flags = 0,
 112        .d_open = t4_open,
 113        .d_close = t4_close,
 114        .d_ioctl = t4_ioctl,
 115        .d_name = "t4nex",
 116 };
 117
 118 /* T5 bus driver interface */
 119 static int t5_probe(device_t);
 120 static device_method_t t5_methods[] = {
 121         DEVMETHOD(device_probe,         t5_probe),
 122         DEVMETHOD(device_attach,        t4_attach),
 123         DEVMETHOD(device_detach,        t4_detach),
 124
 125         DEVMETHOD_END
 126 };
 127 static driver_t t5_driver = {
 128         "t5nex",
 129         t5_methods,
 130         sizeof(struct adapter)
 131 };
 132
 133
 134 /* T5 port (cxl) interface */
 135 static driver_t cxl_driver = {
 136         "cxl",
 137         cxgbe_methods,
 138         sizeof(struct port_info)
 139 };
 140
 141 static struct cdevsw t5_cdevsw = {
 142        .d_version = D_VERSION,
 143        .d_flags = 0,
 144        .d_open = t4_open,
 145        .d_close = t4_close,
 146        .d_ioctl = t4_ioctl,
 147        .d_name = "t5nex",
 148 };
 149
 150 /* ifnet + media interface */
 151 static void cxgbe_init(void *);
 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 154 static void cxgbe_qflush(struct ifnet *);
 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 156 static int cxgbe_media_change(struct ifnet *);
 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 158
 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 160
 161 /*
 162  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 163  * then ADAPTER_LOCK, then t4_uld_list_lock.
 164  */
 165 static struct sx t4_list_lock;
 166 SLIST_HEAD(, adapter) t4_list;
 167 #ifdef TCP_OFFLOAD
 168 static struct sx t4_uld_list_lock;
 169 SLIST_HEAD(, uld_info) t4_uld_list;
 170 #endif
 171
 172 /*
 173  * Tunables.  See tweak_tunables() too.
 174  *
 175  * Each tunable is set to a default value here if it's known at compile-time.
 176  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 177  * provide a reasonable default when the driver is loaded.
 178  *
 179  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 180  * T5 are under hw.cxl.
 181  */
 182
 183 /*
 184  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 185  */
 186 #define NTXQ_10G 16
 187 static int t4_ntxq10g = -1;
 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 189
 190 #define NRXQ_10G 8
 191 static int t4_nrxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 193
 194 #define NTXQ_1G 4
 195 static int t4_ntxq1g = -1;
 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 197
 198 #define NRXQ_1G 2
 199 static int t4_nrxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 201
 202 static int t4_rsrv_noflowq = 0;
 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 204
 205 #ifdef TCP_OFFLOAD
 206 #define NOFLDTXQ_10G 8
 207 static int t4_nofldtxq10g = -1;
 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 209
 210 #define NOFLDRXQ_10G 2
 211 static int t4_nofldrxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 213
 214 #define NOFLDTXQ_1G 2
 215 static int t4_nofldtxq1g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 217
 218 #define NOFLDRXQ_1G 1
 219 static int t4_nofldrxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 221 #endif
 222
 223 #ifdef DEV_NETMAP
 224 #define NNMTXQ_10G 2
 225 static int t4_nnmtxq10g = -1;
 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 227
 228 #define NNMRXQ_10G 2
 229 static int t4_nnmrxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 231
 232 #define NNMTXQ_1G 1
 233 static int t4_nnmtxq1g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 235
 236 #define NNMRXQ_1G 1
 237 static int t4_nnmrxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 239 #endif
 240
 241 /*
 242  * Holdoff parameters for 10G and 1G ports.
 243  */
 244 #define TMR_IDX_10G 1
 245 static int t4_tmr_idx_10g = TMR_IDX_10G;
 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 247
 248 #define PKTC_IDX_10G (-1)
 249 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 251
 252 #define TMR_IDX_1G 1
 253 static int t4_tmr_idx_1g = TMR_IDX_1G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 255
 256 #define PKTC_IDX_1G (-1)
 257 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 259
 260 /*
 261  * Size (# of entries) of each tx and rx queue.
 262  */
 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 265
 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 268
 269 /*
 270  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 271  */
 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 274
 275 /*
 276  * Configuration file.
 277  */
 278 #define DEFAULT_CF      "default"
 279 #define FLASH_CF        "flash"
 280 #define UWIRE_CF        "uwire"
 281 #define FPGA_CF         "fpga"
 282 static char t4_cfg_file[32] = DEFAULT_CF;
 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 284
 285 /*
 286  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 287  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 288  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 289  *            mark or when signalled to do so, 0 to never emit PAUSE.
 290  */
 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 293
 294 /*
 295  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 296  * encouraged respectively).
 297  */
 298 static unsigned int t4_fw_install = 1;
 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 300
 301 /*
 302  * ASIC features that will be used.  Disable the ones you don't want so that the
 303  * chip resources aren't wasted on features that will not be used.
 304  */
 305 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 307
 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 310
 311 static int t4_toecaps_allowed = -1;
 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 313
 314 static int t4_rdmacaps_allowed = 0;
 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 316
 317 static int t4_iscsicaps_allowed = 0;
 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 319
 320 static int t4_fcoecaps_allowed = 0;
 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 322
 323 static int t5_write_combine = 0;
 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 325
 326 struct intrs_and_queues {
 327         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 328         uint16_t nirq;          /* Total # of vectors */
 329         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 330         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 331         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 332         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 333         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 334         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 335         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 336 #ifdef TCP_OFFLOAD
 337         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 338         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 339         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 340         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 341 #endif
 342 #ifdef DEV_NETMAP
 343         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 344         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 345         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 346         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 347 #endif
 348 };
 349
 350 struct filter_entry {
 351         uint32_t valid:1;       /* filter allocated and valid */
 352         uint32_t locked:1;      /* filter is administratively locked */
 353         uint32_t pending:1;     /* filter action is pending firmware reply */
 354         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 355         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 356
 357         struct t4_filter_specification fs;
 358 };
 359
 360 static int map_bars_0_and_4(struct adapter *);
 361 static int map_bar_2(struct adapter *);
 362 static void setup_memwin(struct adapter *);
 363 static int validate_mem_range(struct adapter *, uint32_t, int);
 364 static int fwmtype_to_hwmtype(int);
 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 366     uint32_t *);
 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 368 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 369 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 370     struct intrs_and_queues *);
 371 static int prep_firmware(struct adapter *);
 372 static int partition_resources(struct adapter *, const struct firmware *,
 373     const char *);
 374 static int get_params__pre_init(struct adapter *);
 375 static int get_params__post_init(struct adapter *);
 376 static int set_params__post_init(struct adapter *);
 377 static void t4_set_desc(struct adapter *);
 378 static void build_medialist(struct port_info *, struct ifmedia *);
 379 static int cxgbe_init_synchronized(struct port_info *);
 380 static int cxgbe_uninit_synchronized(struct port_info *);
 381 static int setup_intr_handlers(struct adapter *);
 382 static void quiesce_txq(struct adapter *, struct sge_txq *);
 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 384 static void quiesce_iq(struct adapter *, struct sge_iq *);
 385 static void quiesce_fl(struct adapter *, struct sge_fl *);
 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 387     driver_intr_t *, void *, char *);
 388 static int t4_free_irq(struct adapter *, struct irq *);
 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 390     unsigned int);
 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 393 static void cxgbe_tick(void *);
 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 396     struct mbuf *);
 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 398 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 399 static int t4_sysctls(struct adapter *);
 400 static int cxgbe_sysctls(struct port_info *);
 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 412 #ifdef SBUF_DRAIN
 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 438 #endif
 439 static uint32_t fconf_to_mode(uint32_t);
 440 static uint32_t mode_to_fconf(uint32_t);
 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 442 static int get_filter_mode(struct adapter *, uint32_t *);
 443 static int set_filter_mode(struct adapter *, uint32_t);
 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 445 static int get_filter(struct adapter *, struct t4_filter *);
 446 static int set_filter(struct adapter *, struct t4_filter *);
 447 static int del_filter(struct adapter *, struct t4_filter *);
 448 static void clear_filter(struct filter_entry *);
 449 static int set_filter_wr(struct adapter *, int);
 450 static int del_filter_wr(struct adapter *, int);
 451 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 452 static int load_fw(struct adapter *, struct t4_data *);
 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 454 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 455 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 457 #ifdef TCP_OFFLOAD
 458 static int toe_capability(struct port_info *, int);
 459 #endif
 460 static int mod_event(module_t, int, void *);
 461
 462 struct {
 463         uint16_t device;
 464         char *desc;
 465 } t4_pciids[] = {
 466         {0xa000, "Chelsio Terminator 4 FPGA"},
 467         {0x4400, "Chelsio T440-dbg"},
 468         {0x4401, "Chelsio T420-CR"},
 469         {0x4402, "Chelsio T422-CR"},
 470         {0x4403, "Chelsio T440-CR"},
 471         {0x4404, "Chelsio T420-BCH"},
 472         {0x4405, "Chelsio T440-BCH"},
 473         {0x4406, "Chelsio T440-CH"},
 474         {0x4407, "Chelsio T420-SO"},
 475         {0x4408, "Chelsio T420-CX"},
 476         {0x4409, "Chelsio T420-BT"},
 477         {0x440a, "Chelsio T404-BT"},
 478         {0x440e, "Chelsio T440-LP-CR"},
 479 }, t5_pciids[] = {
 480         {0xb000, "Chelsio Terminator 5 FPGA"},
 481         {0x5400, "Chelsio T580-dbg"},
 482         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 483         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 484         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 485         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 486         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 487         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 488         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 489         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 490         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 491         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 492         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 493         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 494         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 495 #ifdef notyet
 496         {0x5404,  "Chelsio T520-BCH"},
 497         {0x5405,  "Chelsio T540-BCH"},
 498         {0x5406,  "Chelsio T540-CH"},
 499         {0x5408,  "Chelsio T520-CX"},
 500         {0x540b,  "Chelsio B520-SR"},
 501         {0x540c,  "Chelsio B504-BT"},
 502         {0x540f,  "Chelsio Amsterdam"},
 503         {0x5413,  "Chelsio T580-CHR"},
 504 #endif
 505 };
 506
 507 #ifdef TCP_OFFLOAD
 508 /*
 509  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 510  * exactly the same for both rxq and ofld_rxq.
 511  */
 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 514 #endif
 515
 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 519
 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 521
 522 static int
 523 t4_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xa000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t4_pciids); i++) {
 538                 if (d == t4_pciids[i].device) {
 539                         device_set_desc(dev, t4_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t5_probe(device_t dev)
 549 {
 550         int i;
 551         uint16_t v = pci_get_vendor(dev);
 552         uint16_t d = pci_get_device(dev);
 553         uint8_t f = pci_get_function(dev);
 554
 555         if (v != PCI_VENDOR_ID_CHELSIO)
 556                 return (ENXIO);
 557
 558         /* Attach only to PF0 of the FPGA */
 559         if (d == 0xb000 && f != 0)
 560                 return (ENXIO);
 561
 562         for (i = 0; i < nitems(t5_pciids); i++) {
 563                 if (d == t5_pciids[i].device) {
 564                         device_set_desc(dev, t5_pciids[i].desc);
 565                         return (BUS_PROBE_DEFAULT);
 566                 }
 567         }
 568
 569         return (ENXIO);
 570 }
 571
 572 static int
 573 t4_attach(device_t dev)
 574 {
 575         struct adapter *sc;
 576         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 577         struct intrs_and_queues iaq;
 578         struct sge *s;
 579 #ifdef TCP_OFFLOAD
 580         int ofld_rqidx, ofld_tqidx;
 581 #endif
 582 #ifdef DEV_NETMAP
 583         int nm_rqidx, nm_tqidx;
 584 #endif
 585         const char *pcie_ts;
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671         /* Prepare the firmware for operation */
 672         rc = prep_firmware(sc);
 673         if (rc != 0)
 674                 goto done; /* error message displayed already */
 675
 676         rc = get_params__post_init(sc);
 677         if (rc != 0)
 678                 goto done; /* error message displayed already */
 679
 680         rc = set_params__post_init(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = map_bar_2(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = t4_create_dma_tag(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         /*
 693          * First pass over all the ports - allocate VIs and initialize some
 694          * basic parameters like mac address, port type, etc.  We also figure
 695          * out whether a port is 10G or 1G and use that information when
 696          * calculating how many interrupts to attempt to allocate.
 697          */
 698         n10g = n1g = 0;
 699         for_each_port(sc, i) {
 700                 struct port_info *pi;
 701
 702                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 703                 sc->port[i] = pi;
 704
 705                 /* These must be set before t4_port_init */
 706                 pi->adapter = sc;
 707                 pi->port_id = i;
 708
 709                 /* Allocate the vi and initialize parameters like mac addr */
 710                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 711                 if (rc != 0) {
 712                         device_printf(dev, "unable to initialize port %d: %d\n",
 713                             i, rc);
 714                         free(pi, M_CXGBE);
 715                         sc->port[i] = NULL;
 716                         goto done;
 717                 }
 718
 719                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 720                 pi->link_cfg.requested_fc |= t4_pause_settings;
 721                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 722                 pi->link_cfg.fc |= t4_pause_settings;
 723
 724                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 725                 if (rc != 0) {
 726                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 727                         free(pi, M_CXGBE);
 728                         sc->port[i] = NULL;
 729                         goto done;
 730                 }
 731
 732                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 733                     device_get_nameunit(dev), i);
 734                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 735                 sc->chan_map[pi->tx_chan] = i;
 736
 737                 if (is_10G_port(pi) || is_40G_port(pi)) {
 738                         n10g++;
 739                         pi->tmr_idx = t4_tmr_idx_10g;
 740                         pi->pktc_idx = t4_pktc_idx_10g;
 741                 } else {
 742                         n1g++;
 743                         pi->tmr_idx = t4_tmr_idx_1g;
 744                         pi->pktc_idx = t4_pktc_idx_1g;
 745                 }
 746
 747                 pi->xact_addr_filt = -1;
 748                 pi->linkdnrc = -1;
 749
 750                 pi->qsize_rxq = t4_qsize_rxq;
 751                 pi->qsize_txq = t4_qsize_txq;
 752
 753                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 754                 if (pi->dev == NULL) {
 755                         device_printf(dev,
 756                             "failed to add device for port %d.\n", i);
 757                         rc = ENXIO;
 758                         goto done;
 759                 }
 760                 device_set_softc(pi->dev, pi);
 761         }
 762
 763         /*
 764          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 765          */
 766         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 767         if (rc != 0)
 768                 goto done; /* error message displayed already */
 769
 770         sc->intr_type = iaq.intr_type;
 771         sc->intr_count = iaq.nirq;
 772
 773         s = &sc->sge;
 774         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 775         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 776         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 777         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 778         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 779 #ifdef TCP_OFFLOAD
 780         if (is_offload(sc)) {
 781                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 782                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 783                 s->neq += s->nofldtxq + s->nofldrxq;
 784                 s->niq += s->nofldrxq;
 785
 786                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 787                     M_CXGBE, M_ZERO | M_WAITOK);
 788                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 789                     M_CXGBE, M_ZERO | M_WAITOK);
 790         }
 791 #endif
 792 #ifdef DEV_NETMAP
 793         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 794         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 795         s->neq += s->nnmtxq + s->nnmrxq;
 796         s->niq += s->nnmrxq;
 797
 798         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 799             M_CXGBE, M_ZERO | M_WAITOK);
 800         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 801             M_CXGBE, M_ZERO | M_WAITOK);
 802 #endif
 803
 804         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 809             M_ZERO | M_WAITOK);
 810         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 811             M_ZERO | M_WAITOK);
 812         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814
 815         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 816             M_ZERO | M_WAITOK);
 817
 818         t4_init_l2t(sc, M_WAITOK);
 819
 820         /*
 821          * Second pass over the ports.  This time we know the number of rx and
 822          * tx queues that each port should get.
 823          */
 824         rqidx = tqidx = 0;
 825 #ifdef TCP_OFFLOAD
 826         ofld_rqidx = ofld_tqidx = 0;
 827 #endif
 828 #ifdef DEV_NETMAP
 829         nm_rqidx = nm_tqidx = 0;
 830 #endif
 831         for_each_port(sc, i) {
 832                 struct port_info *pi = sc->port[i];
 833
 834                 if (pi == NULL)
 835                         continue;
 836
 837                 pi->first_rxq = rqidx;
 838                 pi->first_txq = tqidx;
 839                 if (is_10G_port(pi) || is_40G_port(pi)) {
 840                         pi->flags |= iaq.intr_flags_10g;
 841                         pi->nrxq = iaq.nrxq10g;
 842                         pi->ntxq = iaq.ntxq10g;
 843                 } else {
 844                         pi->flags |= iaq.intr_flags_1g;
 845                         pi->nrxq = iaq.nrxq1g;
 846                         pi->ntxq = iaq.ntxq1g;
 847                 }
 848
 849                 if (pi->ntxq > 1)
 850                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 851                 else
 852                         pi->rsrv_noflowq = 0;
 853
 854                 rqidx += pi->nrxq;
 855                 tqidx += pi->ntxq;
 856 #ifdef TCP_OFFLOAD
 857                 if (is_offload(sc)) {
 858                         pi->first_ofld_rxq = ofld_rqidx;
 859                         pi->first_ofld_txq = ofld_tqidx;
 860                         if (is_10G_port(pi) || is_40G_port(pi)) {
 861                                 pi->nofldrxq = iaq.nofldrxq10g;
 862                                 pi->nofldtxq = iaq.nofldtxq10g;
 863                         } else {
 864                                 pi->nofldrxq = iaq.nofldrxq1g;
 865                                 pi->nofldtxq = iaq.nofldtxq1g;
 866                         }
 867                         ofld_rqidx += pi->nofldrxq;
 868                         ofld_tqidx += pi->nofldtxq;
 869                 }
 870 #endif
 871 #ifdef DEV_NETMAP
 872                 pi->first_nm_rxq = nm_rqidx;
 873                 pi->first_nm_txq = nm_tqidx;
 874                 if (is_10G_port(pi) || is_40G_port(pi)) {
 875                         pi->nnmrxq = iaq.nnmrxq10g;
 876                         pi->nnmtxq = iaq.nnmtxq10g;
 877                 } else {
 878                         pi->nnmrxq = iaq.nnmrxq1g;
 879                         pi->nnmtxq = iaq.nnmtxq1g;
 880                 }
 881                 nm_rqidx += pi->nnmrxq;
 882                 nm_tqidx += pi->nnmtxq;
 883 #endif
 884         }
 885
 886         rc = setup_intr_handlers(sc);
 887         if (rc != 0) {
 888                 device_printf(dev,
 889                     "failed to setup interrupt handlers: %d\n", rc);
 890                 goto done;
 891         }
 892
 893         rc = bus_generic_attach(dev);
 894         if (rc != 0) {
 895                 device_printf(dev,
 896                     "failed to attach all child ports: %d\n", rc);
 897                 goto done;
 898         }
 899
 900         switch (sc->params.pci.speed) {
 901                 case 0x1:
 902                         pcie_ts = "2.5";
 903                         break;
 904                 case 0x2:
 905                         pcie_ts = "5.0";
 906                         break;
 907                 case 0x3:
 908                         pcie_ts = "8.0";
 909                         break;
 910                 default:
 911                         pcie_ts = "??";
 912                         break;
 913         }
 914         device_printf(dev,
 915             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 916             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 917             sc->params.nports, sc->intr_count,
 918             sc->intr_type == INTR_MSIX ? "MSI-X" :
 919             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 920             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 921
 922         t4_set_desc(sc);
 923
 924 done:
 925         if (rc != 0 && sc->cdev) {
 926                 /* cdev was created and so cxgbetool works; recover that way. */
 927                 device_printf(dev,
 928                     "error during attach, adapter is now in recovery mode.\n");
 929                 rc = 0;
 930         }
 931
 932         if (rc != 0)
 933                 t4_detach(dev);
 934         else
 935                 t4_sysctls(sc);
 936
 937         return (rc);
 938 }
 939
 940 /*
 941  * Idempotent
 942  */
 943 static int
 944 t4_detach(device_t dev)
 945 {
 946         struct adapter *sc;
 947         struct port_info *pi;
 948         int i, rc;
 949
 950         sc = device_get_softc(dev);
 951
 952         if (sc->flags & FULL_INIT_DONE)
 953                 t4_intr_disable(sc);
 954
 955         if (sc->cdev) {
 956                 destroy_dev(sc->cdev);
 957                 sc->cdev = NULL;
 958         }
 959
 960         rc = bus_generic_detach(dev);
 961         if (rc) {
 962                 device_printf(dev,
 963                     "failed to detach child devices: %d\n", rc);
 964                 return (rc);
 965         }
 966
 967         for (i = 0; i < sc->intr_count; i++)
 968                 t4_free_irq(sc, &sc->irq[i]);
 969
 970         for (i = 0; i < MAX_NPORTS; i++) {
 971                 pi = sc->port[i];
 972                 if (pi) {
 973                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 974                         if (pi->dev)
 975                                 device_delete_child(dev, pi->dev);
 976
 977                         mtx_destroy(&pi->pi_lock);
 978                         free(pi, M_CXGBE);
 979                 }
 980         }
 981
 982         if (sc->flags & FULL_INIT_DONE)
 983                 adapter_full_uninit(sc);
 984
 985         if (sc->flags & FW_OK)
 986                 t4_fw_bye(sc, sc->mbox);
 987
 988         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 989                 pci_release_msi(dev);
 990
 991         if (sc->regs_res)
 992                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 993                     sc->regs_res);
 994
 995         if (sc->udbs_res)
 996                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 997                     sc->udbs_res);
 998
 999         if (sc->msix_res)
1000                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1001                     sc->msix_res);
1002
1003         if (sc->l2t)
1004                 t4_free_l2t(sc->l2t);
1005
1006 #ifdef TCP_OFFLOAD
1007         free(sc->sge.ofld_rxq, M_CXGBE);
1008         free(sc->sge.ofld_txq, M_CXGBE);
1009 #endif
1010 #ifdef DEV_NETMAP
1011         free(sc->sge.nm_rxq, M_CXGBE);
1012         free(sc->sge.nm_txq, M_CXGBE);
1013 #endif
1014         free(sc->irq, M_CXGBE);
1015         free(sc->sge.rxq, M_CXGBE);
1016         free(sc->sge.txq, M_CXGBE);
1017         free(sc->sge.ctrlq, M_CXGBE);
1018         free(sc->sge.iqmap, M_CXGBE);
1019         free(sc->sge.eqmap, M_CXGBE);
1020         free(sc->tids.ftid_tab, M_CXGBE);
1021         t4_destroy_dma_tag(sc);
1022         if (mtx_initialized(&sc->sc_lock)) {
1023                 sx_xlock(&t4_list_lock);
1024                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1025                 sx_xunlock(&t4_list_lock);
1026                 mtx_destroy(&sc->sc_lock);
1027         }
1028
1029         if (mtx_initialized(&sc->tids.ftid_lock))
1030                 mtx_destroy(&sc->tids.ftid_lock);
1031         if (mtx_initialized(&sc->sfl_lock))
1032                 mtx_destroy(&sc->sfl_lock);
1033         if (mtx_initialized(&sc->ifp_lock))
1034                 mtx_destroy(&sc->ifp_lock);
1035         if (mtx_initialized(&sc->regwin_lock))
1036                 mtx_destroy(&sc->regwin_lock);
1037
1038         bzero(sc, sizeof(*sc));
1039
1040         return (0);
1041 }
1042
1043 static int
1044 cxgbe_probe(device_t dev)
1045 {
1046         char buf[128];
1047         struct port_info *pi = device_get_softc(dev);
1048
1049         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1050         device_set_desc_copy(dev, buf);
1051
1052         return (BUS_PROBE_DEFAULT);
1053 }
1054
1055 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1056     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1057     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1058 #define T4_CAP_ENABLE (T4_CAP)
1059
1060 static int
1061 cxgbe_attach(device_t dev)
1062 {
1063         struct port_info *pi = device_get_softc(dev);
1064         struct ifnet *ifp;
1065         char *s;
1066         int n, o;
1067
1068         /* Allocate an ifnet and set it up */
1069         ifp = if_alloc(IFT_ETHER);
1070         if (ifp == NULL) {
1071                 device_printf(dev, "Cannot allocate ifnet\n");
1072                 return (ENOMEM);
1073         }
1074         pi->ifp = ifp;
1075         ifp->if_softc = pi;
1076
1077         callout_init(&pi->tick, CALLOUT_MPSAFE);
1078
1079         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1080         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1081
1082         ifp->if_init = cxgbe_init;
1083         ifp->if_ioctl = cxgbe_ioctl;
1084         ifp->if_transmit = cxgbe_transmit;
1085         ifp->if_qflush = cxgbe_qflush;
1086         ifp->if_get_counter = cxgbe_get_counter;
1087
1088         ifp->if_capabilities = T4_CAP;
1089 #ifdef TCP_OFFLOAD
1090         if (is_offload(pi->adapter))
1091                 ifp->if_capabilities |= IFCAP_TOE;
1092 #endif
1093         ifp->if_capenable = T4_CAP_ENABLE;
1094         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1095             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1096
1097         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1098         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1099         ifp->if_hw_tsomaxsegsize = 65536;
1100
1101         /* Initialize ifmedia for this port */
1102         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1103             cxgbe_media_status);
1104         build_medialist(pi, &pi->media);
1105
1106         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1107             EVENTHANDLER_PRI_ANY);
1108
1109         ether_ifattach(ifp, pi->hw_addr);
1110
1111         n = 128;
1112         s = malloc(n, M_CXGBE, M_WAITOK);
1113         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1114         MPASS(n > o);
1115 #ifdef TCP_OFFLOAD
1116         if (is_offload(pi->adapter)) {
1117                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1118                     pi->nofldtxq, pi->nofldrxq);
1119                 MPASS(n > o);
1120         }
1121 #endif
1122 #ifdef DEV_NETMAP
1123         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1124             pi->nnmrxq);
1125         MPASS(n > o);
1126 #endif
1127         device_printf(dev, "%s\n", s);
1128         free(s, M_CXGBE);
1129
1130 #ifdef DEV_NETMAP
1131         /* nm_media handled here to keep implementation private to this file */
1132         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1133             cxgbe_media_status);
1134         build_medialist(pi, &pi->nm_media);
1135         create_netmap_ifnet(pi);        /* logs errors it something fails */
1136 #endif
1137         cxgbe_sysctls(pi);
1138
1139         return (0);
1140 }
1141
1142 static int
1143 cxgbe_detach(device_t dev)
1144 {
1145         struct port_info *pi = device_get_softc(dev);
1146         struct adapter *sc = pi->adapter;
1147         struct ifnet *ifp = pi->ifp;
1148
1149         /* Tell if_ioctl and if_init that the port is going away */
1150         ADAPTER_LOCK(sc);
1151         SET_DOOMED(pi);
1152         wakeup(&sc->flags);
1153         while (IS_BUSY(sc))
1154                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1155         SET_BUSY(sc);
1156 #ifdef INVARIANTS
1157         sc->last_op = "t4detach";
1158         sc->last_op_thr = curthread;
1159 #endif
1160         ADAPTER_UNLOCK(sc);
1161
1162         if (pi->flags & HAS_TRACEQ) {
1163                 sc->traceq = -1;        /* cloner should not create ifnet */
1164                 t4_tracer_port_detach(sc);
1165         }
1166
1167         if (pi->vlan_c)
1168                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1169
1170         PORT_LOCK(pi);
1171         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1172         callout_stop(&pi->tick);
1173         PORT_UNLOCK(pi);
1174         callout_drain(&pi->tick);
1175
1176         /* Let detach proceed even if these fail. */
1177         cxgbe_uninit_synchronized(pi);
1178         port_full_uninit(pi);
1179
1180         ifmedia_removeall(&pi->media);
1181         ether_ifdetach(pi->ifp);
1182         if_free(pi->ifp);
1183
1184 #ifdef DEV_NETMAP
1185         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1186         destroy_netmap_ifnet(pi);
1187 #endif
1188
1189         ADAPTER_LOCK(sc);
1190         CLR_BUSY(sc);
1191         wakeup(&sc->flags);
1192         ADAPTER_UNLOCK(sc);
1193
1194         return (0);
1195 }
1196
1197 static void
1198 cxgbe_init(void *arg)
1199 {
1200         struct port_info *pi = arg;
1201         struct adapter *sc = pi->adapter;
1202
1203         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1204                 return;
1205         cxgbe_init_synchronized(pi);
1206         end_synchronized_op(sc, 0);
1207 }
1208
1209 static int
1210 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1211 {
1212         int rc = 0, mtu, flags, can_sleep;
1213         struct port_info *pi = ifp->if_softc;
1214         struct adapter *sc = pi->adapter;
1215         struct ifreq *ifr = (struct ifreq *)data;
1216         uint32_t mask;
1217
1218         switch (cmd) {
1219         case SIOCSIFMTU:
1220                 mtu = ifr->ifr_mtu;
1221                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1222                         return (EINVAL);
1223
1224                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1225                 if (rc)
1226                         return (rc);
1227                 ifp->if_mtu = mtu;
1228                 if (pi->flags & PORT_INIT_DONE) {
1229                         t4_update_fl_bufsize(ifp);
1230                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1231                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1232                 }
1233                 end_synchronized_op(sc, 0);
1234                 break;
1235
1236         case SIOCSIFFLAGS:
1237                 can_sleep = 0;
1238 redo_sifflags:
1239                 rc = begin_synchronized_op(sc, pi,
1240                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1241                 if (rc)
1242                         return (rc);
1243
1244                 if (ifp->if_flags & IFF_UP) {
1245                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1246                                 flags = pi->if_flags;
1247                                 if ((ifp->if_flags ^ flags) &
1248                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1249                                         if (can_sleep == 1) {
1250                                                 end_synchronized_op(sc, 0);
1251                                                 can_sleep = 0;
1252                                                 goto redo_sifflags;
1253                                         }
1254                                         rc = update_mac_settings(ifp,
1255                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1256                                 }
1257                         } else {
1258                                 if (can_sleep == 0) {
1259                                         end_synchronized_op(sc, LOCK_HELD);
1260                                         can_sleep = 1;
1261                                         goto redo_sifflags;
1262                                 }
1263                                 rc = cxgbe_init_synchronized(pi);
1264                         }
1265                         pi->if_flags = ifp->if_flags;
1266                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1267                         if (can_sleep == 0) {
1268                                 end_synchronized_op(sc, LOCK_HELD);
1269                                 can_sleep = 1;
1270                                 goto redo_sifflags;
1271                         }
1272                         rc = cxgbe_uninit_synchronized(pi);
1273                 }
1274                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1275                 break;
1276
1277         case SIOCADDMULTI:
1278         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1279                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1280                 if (rc)
1281                         return (rc);
1282                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1283                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1284                 end_synchronized_op(sc, LOCK_HELD);
1285                 break;
1286
1287         case SIOCSIFCAP:
1288                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1289                 if (rc)
1290                         return (rc);
1291
1292                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1293                 if (mask & IFCAP_TXCSUM) {
1294                         ifp->if_capenable ^= IFCAP_TXCSUM;
1295                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1296
1297                         if (IFCAP_TSO4 & ifp->if_capenable &&
1298                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1299                                 ifp->if_capenable &= ~IFCAP_TSO4;
1300                                 if_printf(ifp,
1301                                     "tso4 disabled due to -txcsum.\n");
1302                         }
1303                 }
1304                 if (mask & IFCAP_TXCSUM_IPV6) {
1305                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1306                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1307
1308                         if (IFCAP_TSO6 & ifp->if_capenable &&
1309                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1310                                 ifp->if_capenable &= ~IFCAP_TSO6;
1311                                 if_printf(ifp,
1312                                     "tso6 disabled due to -txcsum6.\n");
1313                         }
1314                 }
1315                 if (mask & IFCAP_RXCSUM)
1316                         ifp->if_capenable ^= IFCAP_RXCSUM;
1317                 if (mask & IFCAP_RXCSUM_IPV6)
1318                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1319
1320                 /*
1321                  * Note that we leave CSUM_TSO alone (it is always set).  The
1322                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1323                  * sending a TSO request our way, so it's sufficient to toggle
1324                  * IFCAP_TSOx only.
1325                  */
1326                 if (mask & IFCAP_TSO4) {
1327                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1328                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1329                                 if_printf(ifp, "enable txcsum first.\n");
1330                                 rc = EAGAIN;
1331                                 goto fail;
1332                         }
1333                         ifp->if_capenable ^= IFCAP_TSO4;
1334                 }
1335                 if (mask & IFCAP_TSO6) {
1336                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1337                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1338                                 if_printf(ifp, "enable txcsum6 first.\n");
1339                                 rc = EAGAIN;
1340                                 goto fail;
1341                         }
1342                         ifp->if_capenable ^= IFCAP_TSO6;
1343                 }
1344                 if (mask & IFCAP_LRO) {
1345 #if defined(INET) || defined(INET6)
1346                         int i;
1347                         struct sge_rxq *rxq;
1348
1349                         ifp->if_capenable ^= IFCAP_LRO;
1350                         for_each_rxq(pi, i, rxq) {
1351                                 if (ifp->if_capenable & IFCAP_LRO)
1352                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1353                                 else
1354                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1355                         }
1356 #endif
1357                 }
1358 #ifdef TCP_OFFLOAD
1359                 if (mask & IFCAP_TOE) {
1360                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1361
1362                         rc = toe_capability(pi, enable);
1363                         if (rc != 0)
1364                                 goto fail;
1365
1366                         ifp->if_capenable ^= mask;
1367                 }
1368 #endif
1369                 if (mask & IFCAP_VLAN_HWTAGGING) {
1370                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1371                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1372                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1373                 }
1374                 if (mask & IFCAP_VLAN_MTU) {
1375                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1376
1377                         /* Need to find out how to disable auto-mtu-inflation */
1378                 }
1379                 if (mask & IFCAP_VLAN_HWTSO)
1380                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1381                 if (mask & IFCAP_VLAN_HWCSUM)
1382                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1383
1384 #ifdef VLAN_CAPABILITIES
1385                 VLAN_CAPABILITIES(ifp);
1386 #endif
1387 fail:
1388                 end_synchronized_op(sc, 0);
1389                 break;
1390
1391         case SIOCSIFMEDIA:
1392         case SIOCGIFMEDIA:
1393                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1394                 break;
1395
1396         case SIOCGI2C: {
1397                 struct ifi2creq i2c;
1398
1399                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1400                 if (rc != 0)
1401                         break;
1402                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1403                         rc = EPERM;
1404                         break;
1405                 }
1406                 if (i2c.len > sizeof(i2c.data)) {
1407                         rc = EINVAL;
1408                         break;
1409                 }
1410                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1411                 if (rc)
1412                         return (rc);
1413                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1414                     i2c.offset, i2c.len, &i2c.data[0]);
1415                 end_synchronized_op(sc, 0);
1416                 if (rc == 0)
1417                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1418                 break;
1419         }
1420
1421         default:
1422                 rc = ether_ioctl(ifp, cmd, data);
1423         }
1424
1425         return (rc);
1426 }
1427
1428 static int
1429 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1430 {
1431         struct port_info *pi = ifp->if_softc;
1432         struct adapter *sc = pi->adapter;
1433         struct sge_txq *txq;
1434         void *items[1];
1435         int rc;
1436
1437         M_ASSERTPKTHDR(m);
1438         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1439
1440         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1441                 m_freem(m);
1442                 return (ENETDOWN);
1443         }
1444
1445         rc = parse_pkt(&m);
1446         if (__predict_false(rc != 0)) {
1447                 MPASS(m == NULL);                       /* was freed already */
1448                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1449                 return (rc);
1450         }
1451
1452         /* Select a txq. */
1453         txq = &sc->sge.txq[pi->first_txq];
1454         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1455                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1456                     pi->rsrv_noflowq);
1457
1458         items[0] = m;
1459         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1460         if (__predict_false(rc != 0))
1461                 m_freem(m);
1462
1463         return (rc);
1464 }
1465
1466 static void
1467 cxgbe_qflush(struct ifnet *ifp)
1468 {
1469         struct port_info *pi = ifp->if_softc;
1470         struct sge_txq *txq;
1471         int i;
1472
1473         /* queues do not exist if !PORT_INIT_DONE. */
1474         if (pi->flags & PORT_INIT_DONE) {
1475                 for_each_txq(pi, i, txq) {
1476                         TXQ_LOCK(txq);
1477                         txq->eq.flags &= ~EQ_ENABLED;
1478                         TXQ_UNLOCK(txq);
1479                         while (!mp_ring_is_idle(txq->r)) {
1480                                 mp_ring_check_drainage(txq->r, 0);
1481                                 pause("qflush", 1);
1482                         }
1483                 }
1484         }
1485         if_qflush(ifp);
1486 }
1487
1488 static uint64_t
1489 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1490 {
1491         struct port_info *pi = ifp->if_softc;
1492         struct adapter *sc = pi->adapter;
1493         struct port_stats *s = &pi->stats;
1494
1495         cxgbe_refresh_stats(sc, pi);
1496
1497         switch (c) {
1498         case IFCOUNTER_IPACKETS:
1499                 return (s->rx_frames - s->rx_pause);
1500
1501         case IFCOUNTER_IERRORS:
1502                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1503                     s->rx_fcs_err + s->rx_len_err);
1504
1505         case IFCOUNTER_OPACKETS:
1506                 return (s->tx_frames - s->tx_pause);
1507
1508         case IFCOUNTER_OERRORS:
1509                 return (s->tx_error_frames);
1510
1511         case IFCOUNTER_IBYTES:
1512                 return (s->rx_octets - s->rx_pause * 64);
1513
1514         case IFCOUNTER_OBYTES:
1515                 return (s->tx_octets - s->tx_pause * 64);
1516
1517         case IFCOUNTER_IMCASTS:
1518                 return (s->rx_mcast_frames - s->rx_pause);
1519
1520         case IFCOUNTER_OMCASTS:
1521                 return (s->tx_mcast_frames - s->tx_pause);
1522
1523         case IFCOUNTER_IQDROPS:
1524                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1525                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1526                     s->rx_trunc3 + pi->tnl_cong_drops);
1527
1528         case IFCOUNTER_OQDROPS: {
1529                 uint64_t drops;
1530
1531                 drops = s->tx_drop;
1532                 if (pi->flags & PORT_INIT_DONE) {
1533                         int i;
1534                         struct sge_txq *txq;
1535
1536                         for_each_txq(pi, i, txq)
1537                                 drops += counter_u64_fetch(txq->r->drops);
1538                 }
1539
1540                 return (drops);
1541
1542         }
1543
1544         default:
1545                 return (if_get_counter_default(ifp, c));
1546         }
1547 }
1548
1549 static int
1550 cxgbe_media_change(struct ifnet *ifp)
1551 {
1552         struct port_info *pi = ifp->if_softc;
1553
1554         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1555
1556         return (EOPNOTSUPP);
1557 }
1558
1559 static void
1560 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1561 {
1562         struct port_info *pi = ifp->if_softc;
1563         struct ifmedia *media = NULL;
1564         struct ifmedia_entry *cur;
1565         int speed = pi->link_cfg.speed;
1566 #ifdef INVARIANTS
1567         int data = (pi->port_type << 8) | pi->mod_type;
1568 #endif
1569
1570         if (ifp == pi->ifp)
1571                 media = &pi->media;
1572 #ifdef DEV_NETMAP
1573         else if (ifp == pi->nm_ifp)
1574                 media = &pi->nm_media;
1575 #endif
1576         MPASS(media != NULL);
1577
1578         cur = media->ifm_cur;
1579         MPASS(cur->ifm_data == data);
1580
1581         ifmr->ifm_status = IFM_AVALID;
1582         if (!pi->link_cfg.link_ok)
1583                 return;
1584
1585         ifmr->ifm_status |= IFM_ACTIVE;
1586
1587         /* active and current will differ iff current media is autoselect. */
1588         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1589                 return;
1590
1591         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1592         if (speed == SPEED_10000)
1593                 ifmr->ifm_active |= IFM_10G_T;
1594         else if (speed == SPEED_1000)
1595                 ifmr->ifm_active |= IFM_1000_T;
1596         else if (speed == SPEED_100)
1597                 ifmr->ifm_active |= IFM_100_TX;
1598         else if (speed == SPEED_10)
1599                 ifmr->ifm_active |= IFM_10_T;
1600         else
1601                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1602                             speed));
1603 }
1604
1605 void
1606 t4_fatal_err(struct adapter *sc)
1607 {
1608         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1609         t4_intr_disable(sc);
1610         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1611             device_get_nameunit(sc->dev));
1612 }
1613
1614 static int
1615 map_bars_0_and_4(struct adapter *sc)
1616 {
1617         sc->regs_rid = PCIR_BAR(0);
1618         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1619             &sc->regs_rid, RF_ACTIVE);
1620         if (sc->regs_res == NULL) {
1621                 device_printf(sc->dev, "cannot map registers.\n");
1622                 return (ENXIO);
1623         }
1624         sc->bt = rman_get_bustag(sc->regs_res);
1625         sc->bh = rman_get_bushandle(sc->regs_res);
1626         sc->mmio_len = rman_get_size(sc->regs_res);
1627         setbit(&sc->doorbells, DOORBELL_KDB);
1628
1629         sc->msix_rid = PCIR_BAR(4);
1630         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1631             &sc->msix_rid, RF_ACTIVE);
1632         if (sc->msix_res == NULL) {
1633                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1634                 return (ENXIO);
1635         }
1636
1637         return (0);
1638 }
1639
1640 static int
1641 map_bar_2(struct adapter *sc)
1642 {
1643
1644         /*
1645          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1646          * to map it if RDMA is disabled.
1647          */
1648         if (is_t4(sc) && sc->rdmacaps == 0)
1649                 return (0);
1650
1651         sc->udbs_rid = PCIR_BAR(2);
1652         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1653             &sc->udbs_rid, RF_ACTIVE);
1654         if (sc->udbs_res == NULL) {
1655                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1656                 return (ENXIO);
1657         }
1658         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1659
1660         if (is_t5(sc)) {
1661                 setbit(&sc->doorbells, DOORBELL_UDB);
1662 #if defined(__i386__) || defined(__amd64__)
1663                 if (t5_write_combine) {
1664                         int rc;
1665
1666                         /*
1667                          * Enable write combining on BAR2.  This is the
1668                          * userspace doorbell BAR and is split into 128B
1669                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1670                          * with an egress queue.  The first 64B has the doorbell
1671                          * and the second 64B can be used to submit a tx work
1672                          * request with an implicit doorbell.
1673                          */
1674
1675                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1676                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1677                         if (rc == 0) {
1678                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1679                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1680                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1681                         } else {
1682                                 device_printf(sc->dev,
1683                                     "couldn't enable write combining: %d\n",
1684                                     rc);
1685                         }
1686
1687                         t4_write_reg(sc, A_SGE_STAT_CFG,
1688                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1689                 }
1690 #endif
1691         }
1692
1693         return (0);
1694 }
1695
1696 static const struct memwin t4_memwin[] = {
1697         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1698         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1699         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1700 };
1701
1702 static const struct memwin t5_memwin[] = {
1703         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1704         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1705         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1706 };
1707
1708 static void
1709 setup_memwin(struct adapter *sc)
1710 {
1711         const struct memwin *mw;
1712         int i, n;
1713         uint32_t bar0;
1714
1715         if (is_t4(sc)) {
1716                 /*
1717                  * Read low 32b of bar0 indirectly via the hardware backdoor
1718                  * mechanism.  Works from within PCI passthrough environments
1719                  * too, where rman_get_start() can return a different value.  We
1720                  * need to program the T4 memory window decoders with the actual
1721                  * addresses that will be coming across the PCIe link.
1722                  */
1723                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1724                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1725
1726                 mw = &t4_memwin[0];
1727                 n = nitems(t4_memwin);
1728         } else {
1729                 /* T5 uses the relative offset inside the PCIe BAR */
1730                 bar0 = 0;
1731
1732                 mw = &t5_memwin[0];
1733                 n = nitems(t5_memwin);
1734         }
1735
1736         for (i = 0; i < n; i++, mw++) {
1737                 t4_write_reg(sc,
1738                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1739                     (mw->base + bar0) | V_BIR(0) |
1740                     V_WINDOW(ilog2(mw->aperture) - 10));
1741         }
1742
1743         /* flush */
1744         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1745 }
1746
1747 /*
1748  * Verify that the memory range specified by the addr/len pair is valid and lies
1749  * entirely within a single region (EDCx or MCx).
1750  */
1751 static int
1752 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1753 {
1754         uint32_t em, addr_len, maddr, mlen;
1755
1756         /* Memory can only be accessed in naturally aligned 4 byte units */
1757         if (addr & 3 || len & 3 || len == 0)
1758                 return (EINVAL);
1759
1760         /* Enabled memories */
1761         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1762         if (em & F_EDRAM0_ENABLE) {
1763                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1764                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1765                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1766                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1767                     addr + len <= maddr + mlen)
1768                         return (0);
1769         }
1770         if (em & F_EDRAM1_ENABLE) {
1771                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1772                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1773                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1774                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1775                     addr + len <= maddr + mlen)
1776                         return (0);
1777         }
1778         if (em & F_EXT_MEM_ENABLE) {
1779                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1780                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1781                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1782                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1783                     addr + len <= maddr + mlen)
1784                         return (0);
1785         }
1786         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1787                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1788                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1789                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1790                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1791                     addr + len <= maddr + mlen)
1792                         return (0);
1793         }
1794
1795         return (EFAULT);
1796 }
1797
1798 static int
1799 fwmtype_to_hwmtype(int mtype)
1800 {
1801
1802         switch (mtype) {
1803         case FW_MEMTYPE_EDC0:
1804                 return (MEM_EDC0);
1805         case FW_MEMTYPE_EDC1:
1806                 return (MEM_EDC1);
1807         case FW_MEMTYPE_EXTMEM:
1808                 return (MEM_MC0);
1809         case FW_MEMTYPE_EXTMEM1:
1810                 return (MEM_MC1);
1811         default:
1812                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1813         }
1814 }
1815
1816 /*
1817  * Verify that the memory range specified by the memtype/offset/len pair is
1818  * valid and lies entirely within the memtype specified.  The global address of
1819  * the start of the range is returned in addr.
1820  */
1821 static int
1822 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1823     uint32_t *addr)
1824 {
1825         uint32_t em, addr_len, maddr, mlen;
1826
1827         /* Memory can only be accessed in naturally aligned 4 byte units */
1828         if (off & 3 || len & 3 || len == 0)
1829                 return (EINVAL);
1830
1831         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1832         switch (fwmtype_to_hwmtype(mtype)) {
1833         case MEM_EDC0:
1834                 if (!(em & F_EDRAM0_ENABLE))
1835                         return (EINVAL);
1836                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1837                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1838                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1839                 break;
1840         case MEM_EDC1:
1841                 if (!(em & F_EDRAM1_ENABLE))
1842                         return (EINVAL);
1843                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1844                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1845                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1846                 break;
1847         case MEM_MC:
1848                 if (!(em & F_EXT_MEM_ENABLE))
1849                         return (EINVAL);
1850                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1851                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1852                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1853                 break;
1854         case MEM_MC1:
1855                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1856                         return (EINVAL);
1857                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1858                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1859                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1860                 break;
1861         default:
1862                 return (EINVAL);
1863         }
1864
1865         if (mlen > 0 && off < mlen && off + len <= mlen) {
1866                 *addr = maddr + off;    /* global address */
1867                 return (0);
1868         }
1869
1870         return (EFAULT);
1871 }
1872
1873 static void
1874 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1875 {
1876         const struct memwin *mw;
1877
1878         if (is_t4(sc)) {
1879                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1880                     ("%s: incorrect memwin# (%d)", __func__, win));
1881                 mw = &t4_memwin[win];
1882         } else {
1883                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1884                     ("%s: incorrect memwin# (%d)", __func__, win));
1885                 mw = &t5_memwin[win];
1886         }
1887
1888         if (base != NULL)
1889                 *base = mw->base;
1890         if (aperture != NULL)
1891                 *aperture = mw->aperture;
1892 }
1893
1894 /*
1895  * Positions the memory window such that it can be used to access the specified
1896  * address in the chip's address space.  The return value is the offset of addr
1897  * from the start of the window.
1898  */
1899 static uint32_t
1900 position_memwin(struct adapter *sc, int n, uint32_t addr)
1901 {
1902         uint32_t start, pf;
1903         uint32_t reg;
1904
1905         KASSERT(n >= 0 && n <= 3,
1906             ("%s: invalid window %d.", __func__, n));
1907         KASSERT((addr & 3) == 0,
1908             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1909
1910         if (is_t4(sc)) {
1911                 pf = 0;
1912                 start = addr & ~0xf;    /* start must be 16B aligned */
1913         } else {
1914                 pf = V_PFNUM(sc->pf);
1915                 start = addr & ~0x7f;   /* start must be 128B aligned */
1916         }
1917         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1918
1919         t4_write_reg(sc, reg, start | pf);
1920         t4_read_reg(sc, reg);
1921
1922         return (addr - start);
1923 }
1924
1925 static int
1926 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1927     struct intrs_and_queues *iaq)
1928 {
1929         int rc, itype, navail, nrxq10g, nrxq1g, n;
1930         int nofldrxq10g = 0, nofldrxq1g = 0;
1931         int nnmrxq10g = 0, nnmrxq1g = 0;
1932
1933         bzero(iaq, sizeof(*iaq));
1934
1935         iaq->ntxq10g = t4_ntxq10g;
1936         iaq->ntxq1g = t4_ntxq1g;
1937         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1938         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1939         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1940 #ifdef TCP_OFFLOAD
1941         if (is_offload(sc)) {
1942                 iaq->nofldtxq10g = t4_nofldtxq10g;
1943                 iaq->nofldtxq1g = t4_nofldtxq1g;
1944                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1945                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1946         }
1947 #endif
1948 #ifdef DEV_NETMAP
1949         iaq->nnmtxq10g = t4_nnmtxq10g;
1950         iaq->nnmtxq1g = t4_nnmtxq1g;
1951         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1952         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1953 #endif
1954
1955         for (itype = INTR_MSIX; itype; itype >>= 1) {
1956
1957                 if ((itype & t4_intr_types) == 0)
1958                         continue;       /* not allowed */
1959
1960                 if (itype == INTR_MSIX)
1961                         navail = pci_msix_count(sc->dev);
1962                 else if (itype == INTR_MSI)
1963                         navail = pci_msi_count(sc->dev);
1964                 else
1965                         navail = 1;
1966 restart:
1967                 if (navail == 0)
1968                         continue;
1969
1970                 iaq->intr_type = itype;
1971                 iaq->intr_flags_10g = 0;
1972                 iaq->intr_flags_1g = 0;
1973
1974                 /*
1975                  * Best option: an interrupt vector for errors, one for the
1976                  * firmware event queue, and one for every rxq (NIC, TOE, and
1977                  * netmap).
1978                  */
1979                 iaq->nirq = T4_EXTRA_INTR;
1980                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1981                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1982                 if (iaq->nirq <= navail &&
1983                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1984                         iaq->intr_flags_10g = INTR_ALL;
1985                         iaq->intr_flags_1g = INTR_ALL;
1986                         goto allocate;
1987                 }
1988
1989                 /*
1990                  * Second best option: a vector for errors, one for the firmware
1991                  * event queue, and vectors for either all the NIC rx queues or
1992                  * all the TOE rx queues.  The queues that don't get vectors
1993                  * will forward their interrupts to those that do.
1994                  *
1995                  * Note: netmap rx queues cannot be created early and so they
1996                  * can't be setup to receive forwarded interrupts for others.
1997                  */
1998                 iaq->nirq = T4_EXTRA_INTR;
1999                 if (nrxq10g >= nofldrxq10g) {
2000                         iaq->intr_flags_10g = INTR_RXQ;
2001                         iaq->nirq += n10g * nrxq10g;
2002 #ifdef DEV_NETMAP
2003                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2004 #endif
2005                 } else {
2006                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2007                         iaq->nirq += n10g * nofldrxq10g;
2008 #ifdef DEV_NETMAP
2009                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2010 #endif
2011                 }
2012                 if (nrxq1g >= nofldrxq1g) {
2013                         iaq->intr_flags_1g = INTR_RXQ;
2014                         iaq->nirq += n1g * nrxq1g;
2015 #ifdef DEV_NETMAP
2016                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2017 #endif
2018                 } else {
2019                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2020                         iaq->nirq += n1g * nofldrxq1g;
2021 #ifdef DEV_NETMAP
2022                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2023 #endif
2024                 }
2025                 if (iaq->nirq <= navail &&
2026                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2027                         goto allocate;
2028
2029                 /*
2030                  * Next best option: an interrupt vector for errors, one for the
2031                  * firmware event queue, and at least one per port.  At this
2032                  * point we know we'll have to downsize nrxq and/or nofldrxq
2033                  * and/or nnmrxq to fit what's available to us.
2034                  */
2035                 iaq->nirq = T4_EXTRA_INTR;
2036                 iaq->nirq += n10g + n1g;
2037                 if (iaq->nirq <= navail) {
2038                         int leftover = navail - iaq->nirq;
2039
2040                         if (n10g > 0) {
2041                                 int target = max(nrxq10g, nofldrxq10g);
2042
2043                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2044                                     INTR_RXQ : INTR_OFLD_RXQ;
2045
2046                                 n = 1;
2047                                 while (n < target && leftover >= n10g) {
2048                                         leftover -= n10g;
2049                                         iaq->nirq += n10g;
2050                                         n++;
2051                                 }
2052                                 iaq->nrxq10g = min(n, nrxq10g);
2053 #ifdef TCP_OFFLOAD
2054                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2055 #endif
2056 #ifdef DEV_NETMAP
2057                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2058 #endif
2059                         }
2060
2061                         if (n1g > 0) {
2062                                 int target = max(nrxq1g, nofldrxq1g);
2063
2064                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2065                                     INTR_RXQ : INTR_OFLD_RXQ;
2066
2067                                 n = 1;
2068                                 while (n < target && leftover >= n1g) {
2069                                         leftover -= n1g;
2070                                         iaq->nirq += n1g;
2071                                         n++;
2072                                 }
2073                                 iaq->nrxq1g = min(n, nrxq1g);
2074 #ifdef TCP_OFFLOAD
2075                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2076 #endif
2077 #ifdef DEV_NETMAP
2078                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2079 #endif
2080                         }
2081
2082                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2083                                 goto allocate;
2084                 }
2085
2086                 /*
2087                  * Least desirable option: one interrupt vector for everything.
2088                  */
2089                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2090                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2091 #ifdef TCP_OFFLOAD
2092                 if (is_offload(sc))
2093                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2094 #endif
2095 #ifdef DEV_NETMAP
2096                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2097 #endif
2098
2099 allocate:
2100                 navail = iaq->nirq;
2101                 rc = 0;
2102                 if (itype == INTR_MSIX)
2103                         rc = pci_alloc_msix(sc->dev, &navail);
2104                 else if (itype == INTR_MSI)
2105                         rc = pci_alloc_msi(sc->dev, &navail);
2106
2107                 if (rc == 0) {
2108                         if (navail == iaq->nirq)
2109                                 return (0);
2110
2111                         /*
2112                          * Didn't get the number requested.  Use whatever number
2113                          * the kernel is willing to allocate (it's in navail).
2114                          */
2115                         device_printf(sc->dev, "fewer vectors than requested, "
2116                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2117                             itype, iaq->nirq, navail);
2118                         pci_release_msi(sc->dev);
2119                         goto restart;
2120                 }
2121
2122                 device_printf(sc->dev,
2123                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2124                     itype, rc, iaq->nirq, navail);
2125         }
2126
2127         device_printf(sc->dev,
2128             "failed to find a usable interrupt type.  "
2129             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2130             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2131
2132         return (ENXIO);
2133 }
2134
2135 #define FW_VERSION(chip) ( \
2136     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2137     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2138     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2139     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2140 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2141
2142 struct fw_info {
2143         uint8_t chip;
2144         char *kld_name;
2145         char *fw_mod_name;
2146         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2147 } fw_info[] = {
2148         {
2149                 .chip = CHELSIO_T4,
2150                 .kld_name = "t4fw_cfg",
2151                 .fw_mod_name = "t4fw",
2152                 .fw_hdr = {
2153                         .chip = FW_HDR_CHIP_T4,
2154                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2155                         .intfver_nic = FW_INTFVER(T4, NIC),
2156                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2157                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2158                         .intfver_ri = FW_INTFVER(T4, RI),
2159                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2160                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2161                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2162                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2163                 },
2164         }, {
2165                 .chip = CHELSIO_T5,
2166                 .kld_name = "t5fw_cfg",
2167                 .fw_mod_name = "t5fw",
2168                 .fw_hdr = {
2169                         .chip = FW_HDR_CHIP_T5,
2170                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2171                         .intfver_nic = FW_INTFVER(T5, NIC),
2172                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2173                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2174                         .intfver_ri = FW_INTFVER(T5, RI),
2175                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2176                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2177                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2178                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2179                 },
2180         }
2181 };
2182
2183 static struct fw_info *
2184 find_fw_info(int chip)
2185 {
2186         int i;
2187
2188         for (i = 0; i < nitems(fw_info); i++) {
2189                 if (fw_info[i].chip == chip)
2190                         return (&fw_info[i]);
2191         }
2192         return (NULL);
2193 }
2194
2195 /*
2196  * Is the given firmware API compatible with the one the driver was compiled
2197  * with?
2198  */
2199 static int
2200 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2201 {
2202
2203         /* short circuit if it's the exact same firmware version */
2204         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2205                 return (1);
2206
2207         /*
2208          * XXX: Is this too conservative?  Perhaps I should limit this to the
2209          * features that are supported in the driver.
2210          */
2211 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2212         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2213             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2214             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2215                 return (1);
2216 #undef SAME_INTF
2217
2218         return (0);
2219 }
2220
2221 /*
2222  * The firmware in the KLD is usable, but should it be installed?  This routine
2223  * explains itself in detail if it indicates the KLD firmware should be
2224  * installed.
2225  */
2226 static int
2227 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2228 {
2229         const char *reason;
2230
2231         if (!card_fw_usable) {
2232                 reason = "incompatible or unusable";
2233                 goto install;
2234         }
2235
2236         if (k > c) {
2237                 reason = "older than the version bundled with this driver";
2238                 goto install;
2239         }
2240
2241         if (t4_fw_install == 2 && k != c) {
2242                 reason = "different than the version bundled with this driver";
2243                 goto install;
2244         }
2245
2246         return (0);
2247
2248 install:
2249         if (t4_fw_install == 0) {
2250                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2251                     "but the driver is prohibited from installing a different "
2252                     "firmware on the card.\n",
2253                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2254                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2255
2256                 return (0);
2257         }
2258
2259         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2260             "installing firmware %u.%u.%u.%u on card.\n",
2261             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2262             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2263             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2264             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2265
2266         return (1);
2267 }
2268 /*
2269  * Establish contact with the firmware and determine if we are the master driver
2270  * or not, and whether we are responsible for chip initialization.
2271  */
2272 static int
2273 prep_firmware(struct adapter *sc)
2274 {
2275         const struct firmware *fw = NULL, *default_cfg;
2276         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2277         enum dev_state state;
2278         struct fw_info *fw_info;
2279         struct fw_hdr *card_fw;         /* fw on the card */
2280         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2281         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2282                                            against */
2283
2284         /* Contact firmware. */
2285         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2286         if (rc < 0 || state == DEV_STATE_ERR) {
2287                 rc = -rc;
2288                 device_printf(sc->dev,
2289                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2290                 return (rc);
2291         }
2292         pf = rc;
2293         if (pf == sc->mbox)
2294                 sc->flags |= MASTER_PF;
2295         else if (state == DEV_STATE_UNINIT) {
2296                 /*
2297                  * We didn't get to be the master so we definitely won't be
2298                  * configuring the chip.  It's a bug if someone else hasn't
2299                  * configured it already.
2300                  */
2301                 device_printf(sc->dev, "couldn't be master(%d), "
2302                     "device not already initialized either(%d).\n", rc, state);
2303                 return (EDOOFUS);
2304         }
2305
2306         /* This is the firmware whose headers the driver was compiled against */
2307         fw_info = find_fw_info(chip_id(sc));
2308         if (fw_info == NULL) {
2309                 device_printf(sc->dev,
2310                     "unable to look up firmware information for chip %d.\n",
2311                     chip_id(sc));
2312                 return (EINVAL);
2313         }
2314         drv_fw = &fw_info->fw_hdr;
2315
2316         /*
2317          * The firmware KLD contains many modules.  The KLD name is also the
2318          * name of the module that contains the default config file.
2319          */
2320         default_cfg = firmware_get(fw_info->kld_name);
2321
2322         /* Read the header of the firmware on the card */
2323         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2324         rc = -t4_read_flash(sc, FLASH_FW_START,
2325             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2326         if (rc == 0)
2327                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2328         else {
2329                 device_printf(sc->dev,
2330                     "Unable to read card's firmware header: %d\n", rc);
2331                 card_fw_usable = 0;
2332         }
2333
2334         /* This is the firmware in the KLD */
2335         fw = firmware_get(fw_info->fw_mod_name);
2336         if (fw != NULL) {
2337                 kld_fw = (const void *)fw->data;
2338                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2339         } else {
2340                 kld_fw = NULL;
2341                 kld_fw_usable = 0;
2342         }
2343
2344         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2345             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2346                 /*
2347                  * Common case: the firmware on the card is an exact match and
2348                  * the KLD is an exact match too, or the KLD is
2349                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2350                  * here -- use cxgbetool loadfw if you want to reinstall the
2351                  * same firmware as the one on the card.
2352                  */
2353         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2354             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2355             be32toh(card_fw->fw_ver))) {
2356
2357                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2358                 if (rc != 0) {
2359                         device_printf(sc->dev,
2360                             "failed to install firmware: %d\n", rc);
2361                         goto done;
2362                 }
2363
2364                 /* Installed successfully, update the cached header too. */
2365                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2366                 card_fw_usable = 1;
2367                 need_fw_reset = 0;      /* already reset as part of load_fw */
2368         }
2369
2370         if (!card_fw_usable) {
2371                 uint32_t d, c, k;
2372
2373                 d = ntohl(drv_fw->fw_ver);
2374                 c = ntohl(card_fw->fw_ver);
2375                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2376
2377                 device_printf(sc->dev, "Cannot find a usable firmware: "
2378                     "fw_install %d, chip state %d, "
2379                     "driver compiled with %d.%d.%d.%d, "
2380                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2381                     t4_fw_install, state,
2382                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2383                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2384                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2385                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2386                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2387                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2388                 rc = EINVAL;
2389                 goto done;
2390         }
2391
2392         /* We're using whatever's on the card and it's known to be good. */
2393         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2394         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2395             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2396             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2397             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2398             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2399         t4_get_tp_version(sc, &sc->params.tp_vers);
2400
2401         /* Reset device */
2402         if (need_fw_reset &&
2403             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2404                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2405                 if (rc != ETIMEDOUT && rc != EIO)
2406                         t4_fw_bye(sc, sc->mbox);
2407                 goto done;
2408         }
2409         sc->flags |= FW_OK;
2410
2411         rc = get_params__pre_init(sc);
2412         if (rc != 0)
2413                 goto done; /* error message displayed already */
2414
2415         /* Partition adapter resources as specified in the config file. */
2416         if (state == DEV_STATE_UNINIT) {
2417
2418                 KASSERT(sc->flags & MASTER_PF,
2419                     ("%s: trying to change chip settings when not master.",
2420                     __func__));
2421
2422                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2423                 if (rc != 0)
2424                         goto done;      /* error message displayed already */
2425
2426                 t4_tweak_chip_settings(sc);
2427
2428                 /* get basic stuff going */
2429                 rc = -t4_fw_initialize(sc, sc->mbox);
2430                 if (rc != 0) {
2431                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2432                         goto done;
2433                 }
2434         } else {
2435                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2436                 sc->cfcsum = 0;
2437         }
2438
2439 done:
2440         free(card_fw, M_CXGBE);
2441         if (fw != NULL)
2442                 firmware_put(fw, FIRMWARE_UNLOAD);
2443         if (default_cfg != NULL)
2444                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2445
2446         return (rc);
2447 }
2448
2449 #define FW_PARAM_DEV(param) \
2450         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2451          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2452 #define FW_PARAM_PFVF(param) \
2453         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2454          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2455
2456 /*
2457  * Partition chip resources for use between various PFs, VFs, etc.
2458  */
2459 static int
2460 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2461     const char *name_prefix)
2462 {
2463         const struct firmware *cfg = NULL;
2464         int rc = 0;
2465         struct fw_caps_config_cmd caps;
2466         uint32_t mtype, moff, finicsum, cfcsum;
2467
2468         /*
2469          * Figure out what configuration file to use.  Pick the default config
2470          * file for the card if the user hasn't specified one explicitly.
2471          */
2472         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2473         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2474                 /* Card specific overrides go here. */
2475                 if (pci_get_device(sc->dev) == 0x440a)
2476                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2477                 if (is_fpga(sc))
2478                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2479         }
2480
2481         /*
2482          * We need to load another module if the profile is anything except
2483          * "default" or "flash".
2484          */
2485         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2486             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2487                 char s[32];
2488
2489                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2490                 cfg = firmware_get(s);
2491                 if (cfg == NULL) {
2492                         if (default_cfg != NULL) {
2493                                 device_printf(sc->dev,
2494                                     "unable to load module \"%s\" for "
2495                                     "configuration profile \"%s\", will use "
2496                                     "the default config file instead.\n",
2497                                     s, sc->cfg_file);
2498                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2499                                     "%s", DEFAULT_CF);
2500                         } else {
2501                                 device_printf(sc->dev,
2502                                     "unable to load module \"%s\" for "
2503                                     "configuration profile \"%s\", will use "
2504                                     "the config file on the card's flash "
2505                                     "instead.\n", s, sc->cfg_file);
2506                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2507                                     "%s", FLASH_CF);
2508                         }
2509                 }
2510         }
2511
2512         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2513             default_cfg == NULL) {
2514                 device_printf(sc->dev,
2515                     "default config file not available, will use the config "
2516                     "file on the card's flash instead.\n");
2517                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2518         }
2519
2520         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2521                 u_int cflen, i, n;
2522                 const uint32_t *cfdata;
2523                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2524
2525                 KASSERT(cfg != NULL || default_cfg != NULL,
2526                     ("%s: no config to upload", __func__));
2527
2528                 /*
2529                  * Ask the firmware where it wants us to upload the config file.
2530                  */
2531                 param = FW_PARAM_DEV(CF);
2532                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2533                 if (rc != 0) {
2534                         /* No support for config file?  Shouldn't happen. */
2535                         device_printf(sc->dev,
2536                             "failed to query config file location: %d.\n", rc);
2537                         goto done;
2538                 }
2539                 mtype = G_FW_PARAMS_PARAM_Y(val);
2540                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2541
2542                 /*
2543                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2544                  * useless stuffing/comments at the end of the config file so
2545                  * it's ok to simply throw away the last remaining bytes when
2546                  * the config file is not an exact multiple of 4.  This also
2547                  * helps with the validate_mt_off_len check.
2548                  */
2549                 if (cfg != NULL) {
2550                         cflen = cfg->datasize & ~3;
2551                         cfdata = cfg->data;
2552                 } else {
2553                         cflen = default_cfg->datasize & ~3;
2554                         cfdata = default_cfg->data;
2555                 }
2556
2557                 if (cflen > FLASH_CFG_MAX_SIZE) {
2558                         device_printf(sc->dev,
2559                             "config file too long (%d, max allowed is %d).  "
2560                             "Will try to use the config on the card, if any.\n",
2561                             cflen, FLASH_CFG_MAX_SIZE);
2562                         goto use_config_on_flash;
2563                 }
2564
2565                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2566                 if (rc != 0) {
2567                         device_printf(sc->dev,
2568                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2569                             "Will try to use the config on the card, if any.\n",
2570                             __func__, mtype, moff, cflen, rc);
2571                         goto use_config_on_flash;
2572                 }
2573
2574                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2575                 while (cflen) {
2576                         off = position_memwin(sc, 2, addr);
2577                         n = min(cflen, mw_aperture - off);
2578                         for (i = 0; i < n; i += 4)
2579                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2580                         cflen -= n;
2581                         addr += n;
2582                 }
2583         } else {
2584 use_config_on_flash:
2585                 mtype = FW_MEMTYPE_FLASH;
2586                 moff = t4_flash_cfg_addr(sc);
2587         }
2588
2589         bzero(&caps, sizeof(caps));
2590         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2591             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2592         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2593             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2594             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2595         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2596         if (rc != 0) {
2597                 device_printf(sc->dev,
2598                     "failed to pre-process config file: %d "
2599                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2600                 goto done;
2601         }
2602
2603         finicsum = be32toh(caps.finicsum);
2604         cfcsum = be32toh(caps.cfcsum);
2605         if (finicsum != cfcsum) {
2606                 device_printf(sc->dev,
2607                     "WARNING: config file checksum mismatch: %08x %08x\n",
2608                     finicsum, cfcsum);
2609         }
2610         sc->cfcsum = cfcsum;
2611
2612 #define LIMIT_CAPS(x) do { \
2613         caps.x &= htobe16(t4_##x##_allowed); \
2614 } while (0)
2615
2616         /*
2617          * Let the firmware know what features will (not) be used so it can tune
2618          * things accordingly.
2619          */
2620         LIMIT_CAPS(linkcaps);
2621         LIMIT_CAPS(niccaps);
2622         LIMIT_CAPS(toecaps);
2623         LIMIT_CAPS(rdmacaps);
2624         LIMIT_CAPS(iscsicaps);
2625         LIMIT_CAPS(fcoecaps);
2626 #undef LIMIT_CAPS
2627
2628         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2629             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2630         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2631         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2632         if (rc != 0) {
2633                 device_printf(sc->dev,
2634                     "failed to process config file: %d.\n", rc);
2635         }
2636 done:
2637         if (cfg != NULL)
2638                 firmware_put(cfg, FIRMWARE_UNLOAD);
2639         return (rc);
2640 }
2641
2642 /*
2643  * Retrieve parameters that are needed (or nice to have) very early.
2644  */
2645 static int
2646 get_params__pre_init(struct adapter *sc)
2647 {
2648         int rc;
2649         uint32_t param[2], val[2];
2650         struct fw_devlog_cmd cmd;
2651         struct devlog_params *dlog = &sc->params.devlog;
2652
2653         param[0] = FW_PARAM_DEV(PORTVEC);
2654         param[1] = FW_PARAM_DEV(CCLK);
2655         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2656         if (rc != 0) {
2657                 device_printf(sc->dev,
2658                     "failed to query parameters (pre_init): %d.\n", rc);
2659                 return (rc);
2660         }
2661
2662         sc->params.portvec = val[0];
2663         sc->params.nports = bitcount32(val[0]);
2664         sc->params.vpd.cclk = val[1];
2665
2666         /* Read device log parameters. */
2667         bzero(&cmd, sizeof(cmd));
2668         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2669             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2670         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2671         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2672         if (rc != 0) {
2673                 device_printf(sc->dev,
2674                     "failed to get devlog parameters: %d.\n", rc);
2675                 bzero(dlog, sizeof (*dlog));
2676                 rc = 0; /* devlog isn't critical for device operation */
2677         } else {
2678                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2679                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2680                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2681                 dlog->size = be32toh(cmd.memsize_devlog);
2682         }
2683
2684         return (rc);
2685 }
2686
2687 /*
2688  * Retrieve various parameters that are of interest to the driver.  The device
2689  * has been initialized by the firmware at this point.
2690  */
2691 static int
2692 get_params__post_init(struct adapter *sc)
2693 {
2694         int rc;
2695         uint32_t param[7], val[7];
2696         struct fw_caps_config_cmd caps;
2697
2698         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2699         param[1] = FW_PARAM_PFVF(EQ_START);
2700         param[2] = FW_PARAM_PFVF(FILTER_START);
2701         param[3] = FW_PARAM_PFVF(FILTER_END);
2702         param[4] = FW_PARAM_PFVF(L2T_START);
2703         param[5] = FW_PARAM_PFVF(L2T_END);
2704         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2705         if (rc != 0) {
2706                 device_printf(sc->dev,
2707                     "failed to query parameters (post_init): %d.\n", rc);
2708                 return (rc);
2709         }
2710
2711         sc->sge.iq_start = val[0];
2712         sc->sge.eq_start = val[1];
2713         sc->tids.ftid_base = val[2];
2714         sc->tids.nftids = val[3] - val[2] + 1;
2715         sc->params.ftid_min = val[2];
2716         sc->params.ftid_max = val[3];
2717         sc->vres.l2t.start = val[4];
2718         sc->vres.l2t.size = val[5] - val[4] + 1;
2719         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2720             ("%s: L2 table size (%u) larger than expected (%u)",
2721             __func__, sc->vres.l2t.size, L2T_SIZE));
2722
2723         /* get capabilites */
2724         bzero(&caps, sizeof(caps));
2725         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2726             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2727         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2728         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2729         if (rc != 0) {
2730                 device_printf(sc->dev,
2731                     "failed to get card capabilities: %d.\n", rc);
2732                 return (rc);
2733         }
2734
2735 #define READ_CAPS(x) do { \
2736         sc->x = htobe16(caps.x); \
2737 } while (0)
2738         READ_CAPS(linkcaps);
2739         READ_CAPS(niccaps);
2740         READ_CAPS(toecaps);
2741         READ_CAPS(rdmacaps);
2742         READ_CAPS(iscsicaps);
2743         READ_CAPS(fcoecaps);
2744
2745         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2746                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2747                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2748                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2749                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2750                 if (rc != 0) {
2751                         device_printf(sc->dev,
2752                             "failed to query NIC parameters: %d.\n", rc);
2753                         return (rc);
2754                 }
2755                 sc->tids.etid_base = val[0];
2756                 sc->params.etid_min = val[0];
2757                 sc->tids.netids = val[1] - val[0] + 1;
2758                 sc->params.netids = sc->tids.netids;
2759                 sc->params.eo_wr_cred = val[2];
2760                 sc->params.ethoffload = 1;
2761         }
2762
2763         if (sc->toecaps) {
2764                 /* query offload-related parameters */
2765                 param[0] = FW_PARAM_DEV(NTID);
2766                 param[1] = FW_PARAM_PFVF(SERVER_START);
2767                 param[2] = FW_PARAM_PFVF(SERVER_END);
2768                 param[3] = FW_PARAM_PFVF(TDDP_START);
2769                 param[4] = FW_PARAM_PFVF(TDDP_END);
2770                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2771                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2772                 if (rc != 0) {
2773                         device_printf(sc->dev,
2774                             "failed to query TOE parameters: %d.\n", rc);
2775                         return (rc);
2776                 }
2777                 sc->tids.ntids = val[0];
2778                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2779                 sc->tids.stid_base = val[1];
2780                 sc->tids.nstids = val[2] - val[1] + 1;
2781                 sc->vres.ddp.start = val[3];
2782                 sc->vres.ddp.size = val[4] - val[3] + 1;
2783                 sc->params.ofldq_wr_cred = val[5];
2784                 sc->params.offload = 1;
2785         }
2786         if (sc->rdmacaps) {
2787                 param[0] = FW_PARAM_PFVF(STAG_START);
2788                 param[1] = FW_PARAM_PFVF(STAG_END);
2789                 param[2] = FW_PARAM_PFVF(RQ_START);
2790                 param[3] = FW_PARAM_PFVF(RQ_END);
2791                 param[4] = FW_PARAM_PFVF(PBL_START);
2792                 param[5] = FW_PARAM_PFVF(PBL_END);
2793                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2794                 if (rc != 0) {
2795                         device_printf(sc->dev,
2796                             "failed to query RDMA parameters(1): %d.\n", rc);
2797                         return (rc);
2798                 }
2799                 sc->vres.stag.start = val[0];
2800                 sc->vres.stag.size = val[1] - val[0] + 1;
2801                 sc->vres.rq.start = val[2];
2802                 sc->vres.rq.size = val[3] - val[2] + 1;
2803                 sc->vres.pbl.start = val[4];
2804                 sc->vres.pbl.size = val[5] - val[4] + 1;
2805
2806                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2807                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2808                 param[2] = FW_PARAM_PFVF(CQ_START);
2809                 param[3] = FW_PARAM_PFVF(CQ_END);
2810                 param[4] = FW_PARAM_PFVF(OCQ_START);
2811                 param[5] = FW_PARAM_PFVF(OCQ_END);
2812                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2813                 if (rc != 0) {
2814                         device_printf(sc->dev,
2815                             "failed to query RDMA parameters(2): %d.\n", rc);
2816                         return (rc);
2817                 }
2818                 sc->vres.qp.start = val[0];
2819                 sc->vres.qp.size = val[1] - val[0] + 1;
2820                 sc->vres.cq.start = val[2];
2821                 sc->vres.cq.size = val[3] - val[2] + 1;
2822                 sc->vres.ocq.start = val[4];
2823                 sc->vres.ocq.size = val[5] - val[4] + 1;
2824         }
2825         if (sc->iscsicaps) {
2826                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2827                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2828                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2829                 if (rc != 0) {
2830                         device_printf(sc->dev,
2831                             "failed to query iSCSI parameters: %d.\n", rc);
2832                         return (rc);
2833                 }
2834                 sc->vres.iscsi.start = val[0];
2835                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2836         }
2837
2838         /*
2839          * We've got the params we wanted to query via the firmware.  Now grab
2840          * some others directly from the chip.
2841          */
2842         rc = t4_read_chip_settings(sc);
2843
2844         return (rc);
2845 }
2846
2847 static int
2848 set_params__post_init(struct adapter *sc)
2849 {
2850         uint32_t param, val;
2851
2852         /* ask for encapsulated CPLs */
2853         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2854         val = 1;
2855         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2856
2857         return (0);
2858 }
2859
2860 #undef FW_PARAM_PFVF
2861 #undef FW_PARAM_DEV
2862
2863 static void
2864 t4_set_desc(struct adapter *sc)
2865 {
2866         char buf[128];
2867         struct adapter_params *p = &sc->params;
2868
2869         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2870             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2871             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2872
2873         device_set_desc_copy(sc->dev, buf);
2874 }
2875
2876 static void
2877 build_medialist(struct port_info *pi, struct ifmedia *media)
2878 {
2879         int data, m;
2880
2881         PORT_LOCK(pi);
2882
2883         ifmedia_removeall(media);
2884
2885         m = IFM_ETHER | IFM_FDX;
2886         data = (pi->port_type << 8) | pi->mod_type;
2887
2888         switch(pi->port_type) {
2889         case FW_PORT_TYPE_BT_XFI:
2890                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2891                 break;
2892
2893         case FW_PORT_TYPE_BT_XAUI:
2894                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2895                 /* fall through */
2896
2897         case FW_PORT_TYPE_BT_SGMII:
2898                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2899                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2900                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2901                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2902                 break;
2903
2904         case FW_PORT_TYPE_CX4:
2905                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2906                 ifmedia_set(media, m | IFM_10G_CX4);
2907                 break;
2908
2909         case FW_PORT_TYPE_QSFP_10G:
2910         case FW_PORT_TYPE_SFP:
2911         case FW_PORT_TYPE_FIBER_XFI:
2912         case FW_PORT_TYPE_FIBER_XAUI:
2913                 switch (pi->mod_type) {
2914
2915                 case FW_PORT_MOD_TYPE_LR:
2916                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2917                         ifmedia_set(media, m | IFM_10G_LR);
2918                         break;
2919
2920                 case FW_PORT_MOD_TYPE_SR:
2921                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2922                         ifmedia_set(media, m | IFM_10G_SR);
2923                         break;
2924
2925                 case FW_PORT_MOD_TYPE_LRM:
2926                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2927                         ifmedia_set(media, m | IFM_10G_LRM);
2928                         break;
2929
2930                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2931                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2932                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2933                         ifmedia_set(media, m | IFM_10G_TWINAX);
2934                         break;
2935
2936                 case FW_PORT_MOD_TYPE_NONE:
2937                         m &= ~IFM_FDX;
2938                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2939                         ifmedia_set(media, m | IFM_NONE);
2940                         break;
2941
2942                 case FW_PORT_MOD_TYPE_NA:
2943                 case FW_PORT_MOD_TYPE_ER:
2944                 default:
2945                         device_printf(pi->dev,
2946                             "unknown port_type (%d), mod_type (%d)\n",
2947                             pi->port_type, pi->mod_type);
2948                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2949                         ifmedia_set(media, m | IFM_UNKNOWN);
2950                         break;
2951                 }
2952                 break;
2953
2954         case FW_PORT_TYPE_QSFP:
2955                 switch (pi->mod_type) {
2956
2957                 case FW_PORT_MOD_TYPE_LR:
2958                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2959                         ifmedia_set(media, m | IFM_40G_LR4);
2960                         break;
2961
2962                 case FW_PORT_MOD_TYPE_SR:
2963                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2964                         ifmedia_set(media, m | IFM_40G_SR4);
2965                         break;
2966
2967                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2968                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2969                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2970                         ifmedia_set(media, m | IFM_40G_CR4);
2971                         break;
2972
2973                 case FW_PORT_MOD_TYPE_NONE:
2974                         m &= ~IFM_FDX;
2975                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2976                         ifmedia_set(media, m | IFM_NONE);
2977                         break;
2978
2979                 default:
2980                         device_printf(pi->dev,
2981                             "unknown port_type (%d), mod_type (%d)\n",
2982                             pi->port_type, pi->mod_type);
2983                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2984                         ifmedia_set(media, m | IFM_UNKNOWN);
2985                         break;
2986                 }
2987                 break;
2988
2989         default:
2990                 device_printf(pi->dev,
2991                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2992                     pi->mod_type);
2993                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2994                 ifmedia_set(media, m | IFM_UNKNOWN);
2995                 break;
2996         }
2997
2998         PORT_UNLOCK(pi);
2999 }
3000
3001 #define FW_MAC_EXACT_CHUNK      7
3002
3003 /*
3004  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3005  * indicates which parameters should be programmed (the rest are left alone).
3006  */
3007 int
3008 update_mac_settings(struct ifnet *ifp, int flags)
3009 {
3010         int rc = 0;
3011         struct port_info *pi = ifp->if_softc;
3012         struct adapter *sc = pi->adapter;
3013         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3014         uint16_t viid = 0xffff;
3015         int16_t *xact_addr_filt = NULL;
3016
3017         ASSERT_SYNCHRONIZED_OP(sc);
3018         KASSERT(flags, ("%s: not told what to update.", __func__));
3019
3020         if (ifp == pi->ifp) {
3021                 viid = pi->viid;
3022                 xact_addr_filt = &pi->xact_addr_filt;
3023         }
3024 #ifdef DEV_NETMAP
3025         else if (ifp == pi->nm_ifp) {
3026                 viid = pi->nm_viid;
3027                 xact_addr_filt = &pi->nm_xact_addr_filt;
3028         }
3029 #endif
3030         if (flags & XGMAC_MTU)
3031                 mtu = ifp->if_mtu;
3032
3033         if (flags & XGMAC_PROMISC)
3034                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3035
3036         if (flags & XGMAC_ALLMULTI)
3037                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3038
3039         if (flags & XGMAC_VLANEX)
3040                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3041
3042         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3043                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3044                     1, vlanex, false);
3045                 if (rc) {
3046                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3047                             rc);
3048                         return (rc);
3049                 }
3050         }
3051
3052         if (flags & XGMAC_UCADDR) {
3053                 uint8_t ucaddr[ETHER_ADDR_LEN];
3054
3055                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3056                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3057                     true, true);
3058                 if (rc < 0) {
3059                         rc = -rc;
3060                         if_printf(ifp, "change_mac failed: %d\n", rc);
3061                         return (rc);
3062                 } else {
3063                         *xact_addr_filt = rc;
3064                         rc = 0;
3065                 }
3066         }
3067
3068         if (flags & XGMAC_MCADDRS) {
3069                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3070                 int del = 1;
3071                 uint64_t hash = 0;
3072                 struct ifmultiaddr *ifma;
3073                 int i = 0, j;
3074
3075                 if_maddr_rlock(ifp);
3076                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3077                         if (ifma->ifma_addr->sa_family != AF_LINK)
3078                                 continue;
3079                         mcaddr[i] =
3080                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3081                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3082                         i++;
3083
3084                         if (i == FW_MAC_EXACT_CHUNK) {
3085                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3086                                     i, mcaddr, NULL, &hash, 0);
3087                                 if (rc < 0) {
3088                                         rc = -rc;
3089                                         for (j = 0; j < i; j++) {
3090                                                 if_printf(ifp,
3091                                                     "failed to add mc address"
3092                                                     " %02x:%02x:%02x:"
3093                                                     "%02x:%02x:%02x rc=%d\n",
3094                                                     mcaddr[j][0], mcaddr[j][1],
3095                                                     mcaddr[j][2], mcaddr[j][3],
3096                                                     mcaddr[j][4], mcaddr[j][5],
3097                                                     rc);
3098                                         }
3099                                         goto mcfail;
3100                                 }
3101                                 del = 0;
3102                                 i = 0;
3103                         }
3104                 }
3105                 if (i > 0) {
3106                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3107                             mcaddr, NULL, &hash, 0);
3108                         if (rc < 0) {
3109                                 rc = -rc;
3110                                 for (j = 0; j < i; j++) {
3111                                         if_printf(ifp,
3112                                             "failed to add mc address"
3113                                             " %02x:%02x:%02x:"
3114                                             "%02x:%02x:%02x rc=%d\n",
3115                                             mcaddr[j][0], mcaddr[j][1],
3116                                             mcaddr[j][2], mcaddr[j][3],
3117                                             mcaddr[j][4], mcaddr[j][5],
3118                                             rc);
3119                                 }
3120                                 goto mcfail;
3121                         }
3122                 }
3123
3124                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3125                 if (rc != 0)
3126                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3127 mcfail:
3128                 if_maddr_runlock(ifp);
3129         }
3130
3131         return (rc);
3132 }
3133
3134 int
3135 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3136     char *wmesg)
3137 {
3138         int rc, pri;
3139
3140 #ifdef WITNESS
3141         /* the caller thinks it's ok to sleep, but is it really? */
3142         if (flags & SLEEP_OK)
3143                 pause("t4slptst", 1);
3144 #endif
3145
3146         if (INTR_OK)
3147                 pri = PCATCH;
3148         else
3149                 pri = 0;
3150
3151         ADAPTER_LOCK(sc);
3152         for (;;) {
3153
3154                 if (pi && IS_DOOMED(pi)) {
3155                         rc = ENXIO;
3156                         goto done;
3157                 }
3158
3159                 if (!IS_BUSY(sc)) {
3160                         rc = 0;
3161                         break;
3162                 }
3163
3164                 if (!(flags & SLEEP_OK)) {
3165                         rc = EBUSY;
3166                         goto done;
3167                 }
3168
3169                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3170                         rc = EINTR;
3171                         goto done;
3172                 }
3173         }
3174
3175         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3176         SET_BUSY(sc);
3177 #ifdef INVARIANTS
3178         sc->last_op = wmesg;
3179         sc->last_op_thr = curthread;
3180 #endif
3181
3182 done:
3183         if (!(flags & HOLD_LOCK) || rc)
3184                 ADAPTER_UNLOCK(sc);
3185
3186         return (rc);
3187 }
3188
3189 void
3190 end_synchronized_op(struct adapter *sc, int flags)
3191 {
3192
3193         if (flags & LOCK_HELD)
3194                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3195         else
3196                 ADAPTER_LOCK(sc);
3197
3198         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3199         CLR_BUSY(sc);
3200         wakeup(&sc->flags);
3201         ADAPTER_UNLOCK(sc);
3202 }
3203
3204 static int
3205 cxgbe_init_synchronized(struct port_info *pi)
3206 {
3207         struct adapter *sc = pi->adapter;
3208         struct ifnet *ifp = pi->ifp;
3209         int rc = 0, i;
3210         struct sge_txq *txq;
3211
3212         ASSERT_SYNCHRONIZED_OP(sc);
3213
3214         if (isset(&sc->open_device_map, pi->port_id)) {
3215                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3216                     ("mismatch between open_device_map and if_drv_flags"));
3217                 return (0);     /* already running */
3218         }
3219
3220         if (!(sc->flags & FULL_INIT_DONE) &&
3221             ((rc = adapter_full_init(sc)) != 0))
3222                 return (rc);    /* error message displayed already */
3223
3224         if (!(pi->flags & PORT_INIT_DONE) &&
3225             ((rc = port_full_init(pi)) != 0))
3226                 return (rc); /* error message displayed already */
3227
3228         rc = update_mac_settings(ifp, XGMAC_ALL);
3229         if (rc)
3230                 goto done;      /* error message displayed already */
3231
3232         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3233         if (rc != 0) {
3234                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3235                 goto done;
3236         }
3237
3238         /*
3239          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3240          * if this changes.
3241          */
3242
3243         for_each_txq(pi, i, txq) {
3244                 TXQ_LOCK(txq);
3245                 txq->eq.flags |= EQ_ENABLED;
3246                 TXQ_UNLOCK(txq);
3247         }
3248
3249         /*
3250          * The first iq of the first port to come up is used for tracing.
3251          */
3252         if (sc->traceq < 0) {
3253                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3254                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3255                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3256                     V_QUEUENUMBER(sc->traceq));
3257                 pi->flags |= HAS_TRACEQ;
3258         }
3259
3260         /* all ok */
3261         setbit(&sc->open_device_map, pi->port_id);
3262         PORT_LOCK(pi);
3263         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3264         PORT_UNLOCK(pi);
3265
3266         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3267 done:
3268         if (rc != 0)
3269                 cxgbe_uninit_synchronized(pi);
3270
3271         return (rc);
3272 }
3273
3274 /*
3275  * Idempotent.
3276  */
3277 static int
3278 cxgbe_uninit_synchronized(struct port_info *pi)
3279 {
3280         struct adapter *sc = pi->adapter;
3281         struct ifnet *ifp = pi->ifp;
3282         int rc, i;
3283         struct sge_txq *txq;
3284
3285         ASSERT_SYNCHRONIZED_OP(sc);
3286
3287         /*
3288          * Disable the VI so that all its data in either direction is discarded
3289          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3290          * tick) intact as the TP can deliver negative advice or data that it's
3291          * holding in its RAM (for an offloaded connection) even after the VI is
3292          * disabled.
3293          */
3294         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3295         if (rc) {
3296                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3297                 return (rc);
3298         }
3299
3300         for_each_txq(pi, i, txq) {
3301                 TXQ_LOCK(txq);
3302                 txq->eq.flags &= ~EQ_ENABLED;
3303                 TXQ_UNLOCK(txq);
3304         }
3305
3306         clrbit(&sc->open_device_map, pi->port_id);
3307         PORT_LOCK(pi);
3308         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3309         PORT_UNLOCK(pi);
3310
3311         pi->link_cfg.link_ok = 0;
3312         pi->link_cfg.speed = 0;
3313         pi->linkdnrc = -1;
3314         t4_os_link_changed(sc, pi->port_id, 0, -1);
3315
3316         return (0);
3317 }
3318
3319 /*
3320  * It is ok for this function to fail midway and return right away.  t4_detach
3321  * will walk the entire sc->irq list and clean up whatever is valid.
3322  */
3323 static int
3324 setup_intr_handlers(struct adapter *sc)
3325 {
3326         int rc, rid, p, q;
3327         char s[8];
3328         struct irq *irq;
3329         struct port_info *pi;
3330         struct sge_rxq *rxq;
3331 #ifdef TCP_OFFLOAD
3332         struct sge_ofld_rxq *ofld_rxq;
3333 #endif
3334 #ifdef DEV_NETMAP
3335         struct sge_nm_rxq *nm_rxq;
3336 #endif
3337
3338         /*
3339          * Setup interrupts.
3340          */
3341         irq = &sc->irq[0];
3342         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3343         if (sc->intr_count == 1)
3344                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3345
3346         /* Multiple interrupts. */
3347         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3348             ("%s: too few intr.", __func__));
3349
3350         /* The first one is always error intr */
3351         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3352         if (rc != 0)
3353                 return (rc);
3354         irq++;
3355         rid++;
3356
3357         /* The second one is always the firmware event queue */
3358         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3359         if (rc != 0)
3360                 return (rc);
3361         irq++;
3362         rid++;
3363
3364         for_each_port(sc, p) {
3365                 pi = sc->port[p];
3366
3367                 if (pi->flags & INTR_RXQ) {
3368                         for_each_rxq(pi, q, rxq) {
3369                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3370                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3371                                     s);
3372                                 if (rc != 0)
3373                                         return (rc);
3374                                 irq++;
3375                                 rid++;
3376                         }
3377                 }
3378 #ifdef TCP_OFFLOAD
3379                 if (pi->flags & INTR_OFLD_RXQ) {
3380                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3381                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3382                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3383                                     ofld_rxq, s);
3384                                 if (rc != 0)
3385                                         return (rc);
3386                                 irq++;
3387                                 rid++;
3388                         }
3389                 }
3390 #endif
3391 #ifdef DEV_NETMAP
3392                 if (pi->flags & INTR_NM_RXQ) {
3393                         for_each_nm_rxq(pi, q, nm_rxq) {
3394                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3395                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3396                                     nm_rxq, s);
3397                                 if (rc != 0)
3398                                         return (rc);
3399                                 irq++;
3400                                 rid++;
3401                         }
3402                 }
3403 #endif
3404         }
3405         MPASS(irq == &sc->irq[sc->intr_count]);
3406
3407         return (0);
3408 }
3409
3410 int
3411 adapter_full_init(struct adapter *sc)
3412 {
3413         int rc, i;
3414
3415         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3416         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3417             ("%s: FULL_INIT_DONE already", __func__));
3418
3419         /*
3420          * queues that belong to the adapter (not any particular port).
3421          */
3422         rc = t4_setup_adapter_queues(sc);
3423         if (rc != 0)
3424                 goto done;
3425
3426         for (i = 0; i < nitems(sc->tq); i++) {
3427                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3428                     taskqueue_thread_enqueue, &sc->tq[i]);
3429                 if (sc->tq[i] == NULL) {
3430                         device_printf(sc->dev,
3431                             "failed to allocate task queue %d\n", i);
3432                         rc = ENOMEM;
3433                         goto done;
3434                 }
3435                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3436                     device_get_nameunit(sc->dev), i);
3437         }
3438
3439         t4_intr_enable(sc);
3440         sc->flags |= FULL_INIT_DONE;
3441 done:
3442         if (rc != 0)
3443                 adapter_full_uninit(sc);
3444
3445         return (rc);
3446 }
3447
3448 int
3449 adapter_full_uninit(struct adapter *sc)
3450 {
3451         int i;
3452
3453         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3454
3455         t4_teardown_adapter_queues(sc);
3456
3457         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3458                 taskqueue_free(sc->tq[i]);
3459                 sc->tq[i] = NULL;
3460         }
3461
3462         sc->flags &= ~FULL_INIT_DONE;
3463
3464         return (0);
3465 }
3466
3467 int
3468 port_full_init(struct port_info *pi)
3469 {
3470         struct adapter *sc = pi->adapter;
3471         struct ifnet *ifp = pi->ifp;
3472         uint16_t *rss;
3473         struct sge_rxq *rxq;
3474         int rc, i, j;
3475
3476         ASSERT_SYNCHRONIZED_OP(sc);
3477         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3478             ("%s: PORT_INIT_DONE already", __func__));
3479
3480         sysctl_ctx_init(&pi->ctx);
3481         pi->flags |= PORT_SYSCTL_CTX;
3482
3483         /*
3484          * Allocate tx/rx/fl queues for this port.
3485          */
3486         rc = t4_setup_port_queues(pi);
3487         if (rc != 0)
3488                 goto done;      /* error message displayed already */
3489
3490         /*
3491          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3492          */
3493         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3494         for (i = 0; i < pi->rss_size;) {
3495                 for_each_rxq(pi, j, rxq) {
3496                         rss[i++] = rxq->iq.abs_id;
3497                         if (i == pi->rss_size)
3498                                 break;
3499                 }
3500         }
3501
3502         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3503             pi->rss_size);
3504         if (rc != 0) {
3505                 if_printf(ifp, "rss_config failed: %d\n", rc);
3506                 goto done;
3507         }
3508
3509         pi->rss = rss;
3510         pi->flags |= PORT_INIT_DONE;
3511 done:
3512         if (rc != 0)
3513                 port_full_uninit(pi);
3514
3515         return (rc);
3516 }
3517
3518 /*
3519  * Idempotent.
3520  */
3521 int
3522 port_full_uninit(struct port_info *pi)
3523 {
3524         struct adapter *sc = pi->adapter;
3525         int i;
3526         struct sge_rxq *rxq;
3527         struct sge_txq *txq;
3528 #ifdef TCP_OFFLOAD
3529         struct sge_ofld_rxq *ofld_rxq;
3530         struct sge_wrq *ofld_txq;
3531 #endif
3532
3533         if (pi->flags & PORT_INIT_DONE) {
3534
3535                 /* Need to quiesce queues.  */
3536
3537                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3538
3539                 for_each_txq(pi, i, txq) {
3540                         quiesce_txq(sc, txq);
3541                 }
3542
3543 #ifdef TCP_OFFLOAD
3544                 for_each_ofld_txq(pi, i, ofld_txq) {
3545                         quiesce_wrq(sc, ofld_txq);
3546                 }
3547 #endif
3548
3549                 for_each_rxq(pi, i, rxq) {
3550                         quiesce_iq(sc, &rxq->iq);
3551                         quiesce_fl(sc, &rxq->fl);
3552                 }
3553
3554 #ifdef TCP_OFFLOAD
3555                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3556                         quiesce_iq(sc, &ofld_rxq->iq);
3557                         quiesce_fl(sc, &ofld_rxq->fl);
3558                 }
3559 #endif
3560                 free(pi->rss, M_CXGBE);
3561         }
3562
3563         t4_teardown_port_queues(pi);
3564         pi->flags &= ~PORT_INIT_DONE;
3565
3566         return (0);
3567 }
3568
3569 static void
3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3571 {
3572         struct sge_eq *eq = &txq->eq;
3573         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3574
3575         (void) sc;      /* unused */
3576
3577 #ifdef INVARIANTS
3578         TXQ_LOCK(txq);
3579         MPASS((eq->flags & EQ_ENABLED) == 0);
3580         TXQ_UNLOCK(txq);
3581 #endif
3582
3583         /* Wait for the mp_ring to empty. */
3584         while (!mp_ring_is_idle(txq->r)) {
3585                 mp_ring_check_drainage(txq->r, 0);
3586                 pause("rquiesce", 1);
3587         }
3588
3589         /* Then wait for the hardware to finish. */
3590         while (spg->cidx != htobe16(eq->pidx))
3591                 pause("equiesce", 1);
3592
3593         /* Finally, wait for the driver to reclaim all descriptors. */
3594         while (eq->cidx != eq->pidx)
3595                 pause("dquiesce", 1);
3596 }
3597
3598 static void
3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3600 {
3601
3602         /* XXXTX */
3603 }
3604
3605 static void
3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3607 {
3608         (void) sc;      /* unused */
3609
3610         /* Synchronize with the interrupt handler */
3611         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3612                 pause("iqfree", 1);
3613 }
3614
3615 static void
3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3617 {
3618         mtx_lock(&sc->sfl_lock);
3619         FL_LOCK(fl);
3620         fl->flags |= FL_DOOMED;
3621         FL_UNLOCK(fl);
3622         mtx_unlock(&sc->sfl_lock);
3623
3624         callout_drain(&sc->sfl_callout);
3625         KASSERT((fl->flags & FL_STARVING) == 0,
3626             ("%s: still starving", __func__));
3627 }
3628
3629 static int
3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3631     driver_intr_t *handler, void *arg, char *name)
3632 {
3633         int rc;
3634
3635         irq->rid = rid;
3636         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3637             RF_SHAREABLE | RF_ACTIVE);
3638         if (irq->res == NULL) {
3639                 device_printf(sc->dev,
3640                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3641                 return (ENOMEM);
3642         }
3643
3644         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3645             NULL, handler, arg, &irq->tag);
3646         if (rc != 0) {
3647                 device_printf(sc->dev,
3648                     "failed to setup interrupt for rid %d, name %s: %d\n",
3649                     rid, name, rc);
3650         } else if (name)
3651                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3652
3653         return (rc);
3654 }
3655
3656 static int
3657 t4_free_irq(struct adapter *sc, struct irq *irq)
3658 {
3659         if (irq->tag)
3660                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3661         if (irq->res)
3662                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3663
3664         bzero(irq, sizeof(*irq));
3665
3666         return (0);
3667 }
3668
3669 static void
3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3671     unsigned int end)
3672 {
3673         uint32_t *p = (uint32_t *)(buf + start);
3674
3675         for ( ; start <= end; start += sizeof(uint32_t))
3676                 *p++ = t4_read_reg(sc, start);
3677 }
3678
3679 static void
3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3681 {
3682         int i, n;
3683         const unsigned int *reg_ranges;
3684         static const unsigned int t4_reg_ranges[] = {
3685                 0x1008, 0x1108,
3686                 0x1180, 0x11b4,
3687                 0x11fc, 0x123c,
3688                 0x1300, 0x173c,
3689                 0x1800, 0x18fc,
3690                 0x3000, 0x30d8,
3691                 0x30e0, 0x5924,
3692                 0x5960, 0x59d4,
3693                 0x5a00, 0x5af8,
3694                 0x6000, 0x6098,
3695                 0x6100, 0x6150,
3696                 0x6200, 0x6208,
3697                 0x6240, 0x6248,
3698                 0x6280, 0x6338,
3699                 0x6370, 0x638c,
3700                 0x6400, 0x643c,
3701                 0x6500, 0x6524,
3702                 0x6a00, 0x6a38,
3703                 0x6a60, 0x6a78,
3704                 0x6b00, 0x6b84,
3705                 0x6bf0, 0x6c84,
3706                 0x6cf0, 0x6d84,
3707                 0x6df0, 0x6e84,
3708                 0x6ef0, 0x6f84,
3709                 0x6ff0, 0x7084,
3710                 0x70f0, 0x7184,
3711                 0x71f0, 0x7284,
3712                 0x72f0, 0x7384,
3713                 0x73f0, 0x7450,
3714                 0x7500, 0x7530,
3715                 0x7600, 0x761c,
3716                 0x7680, 0x76cc,
3717                 0x7700, 0x7798,
3718                 0x77c0, 0x77fc,
3719                 0x7900, 0x79fc,
3720                 0x7b00, 0x7c38,
3721                 0x7d00, 0x7efc,
3722                 0x8dc0, 0x8e1c,
3723                 0x8e30, 0x8e78,
3724                 0x8ea0, 0x8f6c,
3725                 0x8fc0, 0x9074,
3726                 0x90fc, 0x90fc,
3727                 0x9400, 0x9458,
3728                 0x9600, 0x96bc,
3729                 0x9800, 0x9808,
3730                 0x9820, 0x983c,
3731                 0x9850, 0x9864,
3732                 0x9c00, 0x9c6c,
3733                 0x9c80, 0x9cec,
3734                 0x9d00, 0x9d6c,
3735                 0x9d80, 0x9dec,
3736                 0x9e00, 0x9e6c,
3737                 0x9e80, 0x9eec,
3738                 0x9f00, 0x9f6c,
3739                 0x9f80, 0x9fec,
3740                 0xd004, 0xd03c,
3741                 0xdfc0, 0xdfe0,
3742                 0xe000, 0xea7c,
3743                 0xf000, 0x11110,
3744                 0x11118, 0x11190,
3745                 0x19040, 0x1906c,
3746                 0x19078, 0x19080,
3747                 0x1908c, 0x19124,
3748                 0x19150, 0x191b0,
3749                 0x191d0, 0x191e8,
3750                 0x19238, 0x1924c,
3751                 0x193f8, 0x19474,
3752                 0x19490, 0x194f8,
3753                 0x19800, 0x19f30,
3754                 0x1a000, 0x1a06c,
3755                 0x1a0b0, 0x1a120,
3756                 0x1a128, 0x1a138,
3757                 0x1a190, 0x1a1c4,
3758                 0x1a1fc, 0x1a1fc,
3759                 0x1e040, 0x1e04c,
3760                 0x1e284, 0x1e28c,
3761                 0x1e2c0, 0x1e2c0,
3762                 0x1e2e0, 0x1e2e0,
3763                 0x1e300, 0x1e384,
3764                 0x1e3c0, 0x1e3c8,
3765                 0x1e440, 0x1e44c,
3766                 0x1e684, 0x1e68c,
3767                 0x1e6c0, 0x1e6c0,
3768                 0x1e6e0, 0x1e6e0,
3769                 0x1e700, 0x1e784,
3770                 0x1e7c0, 0x1e7c8,
3771                 0x1e840, 0x1e84c,
3772                 0x1ea84, 0x1ea8c,
3773                 0x1eac0, 0x1eac0,
3774                 0x1eae0, 0x1eae0,
3775                 0x1eb00, 0x1eb84,
3776                 0x1ebc0, 0x1ebc8,
3777                 0x1ec40, 0x1ec4c,
3778                 0x1ee84, 0x1ee8c,
3779                 0x1eec0, 0x1eec0,
3780                 0x1eee0, 0x1eee0,
3781                 0x1ef00, 0x1ef84,
3782                 0x1efc0, 0x1efc8,
3783                 0x1f040, 0x1f04c,
3784                 0x1f284, 0x1f28c,
3785                 0x1f2c0, 0x1f2c0,
3786                 0x1f2e0, 0x1f2e0,
3787                 0x1f300, 0x1f384,
3788                 0x1f3c0, 0x1f3c8,
3789                 0x1f440, 0x1f44c,
3790                 0x1f684, 0x1f68c,
3791                 0x1f6c0, 0x1f6c0,
3792                 0x1f6e0, 0x1f6e0,
3793                 0x1f700, 0x1f784,
3794                 0x1f7c0, 0x1f7c8,
3795                 0x1f840, 0x1f84c,
3796                 0x1fa84, 0x1fa8c,
3797                 0x1fac0, 0x1fac0,
3798                 0x1fae0, 0x1fae0,
3799                 0x1fb00, 0x1fb84,
3800                 0x1fbc0, 0x1fbc8,
3801                 0x1fc40, 0x1fc4c,
3802                 0x1fe84, 0x1fe8c,
3803                 0x1fec0, 0x1fec0,
3804                 0x1fee0, 0x1fee0,
3805                 0x1ff00, 0x1ff84,
3806                 0x1ffc0, 0x1ffc8,
3807                 0x20000, 0x2002c,
3808                 0x20100, 0x2013c,
3809                 0x20190, 0x201c8,
3810                 0x20200, 0x20318,
3811                 0x20400, 0x20528,
3812                 0x20540, 0x20614,
3813                 0x21000, 0x21040,
3814                 0x2104c, 0x21060,
3815                 0x210c0, 0x210ec,
3816                 0x21200, 0x21268,
3817                 0x21270, 0x21284,
3818                 0x212fc, 0x21388,
3819                 0x21400, 0x21404,
3820                 0x21500, 0x21518,
3821                 0x2152c, 0x2153c,
3822                 0x21550, 0x21554,
3823                 0x21600, 0x21600,
3824                 0x21608, 0x21628,
3825                 0x21630, 0x2163c,
3826                 0x21700, 0x2171c,
3827                 0x21780, 0x2178c,
3828                 0x21800, 0x21c38,
3829                 0x21c80, 0x21d7c,
3830                 0x21e00, 0x21e04,
3831                 0x22000, 0x2202c,
3832                 0x22100, 0x2213c,
3833                 0x22190, 0x221c8,
3834                 0x22200, 0x22318,
3835                 0x22400, 0x22528,
3836                 0x22540, 0x22614,
3837                 0x23000, 0x23040,
3838                 0x2304c, 0x23060,
3839                 0x230c0, 0x230ec,
3840                 0x23200, 0x23268,
3841                 0x23270, 0x23284,
3842                 0x232fc, 0x23388,
3843                 0x23400, 0x23404,
3844                 0x23500, 0x23518,
3845                 0x2352c, 0x2353c,
3846                 0x23550, 0x23554,
3847                 0x23600, 0x23600,
3848                 0x23608, 0x23628,
3849                 0x23630, 0x2363c,
3850                 0x23700, 0x2371c,
3851                 0x23780, 0x2378c,
3852                 0x23800, 0x23c38,
3853                 0x23c80, 0x23d7c,
3854                 0x23e00, 0x23e04,
3855                 0x24000, 0x2402c,
3856                 0x24100, 0x2413c,
3857                 0x24190, 0x241c8,
3858                 0x24200, 0x24318,
3859                 0x24400, 0x24528,
3860                 0x24540, 0x24614,
3861                 0x25000, 0x25040,
3862                 0x2504c, 0x25060,
3863                 0x250c0, 0x250ec,
3864                 0x25200, 0x25268,
3865                 0x25270, 0x25284,
3866                 0x252fc, 0x25388,
3867                 0x25400, 0x25404,
3868                 0x25500, 0x25518,
3869                 0x2552c, 0x2553c,
3870                 0x25550, 0x25554,
3871                 0x25600, 0x25600,
3872                 0x25608, 0x25628,
3873                 0x25630, 0x2563c,
3874                 0x25700, 0x2571c,
3875                 0x25780, 0x2578c,
3876                 0x25800, 0x25c38,
3877                 0x25c80, 0x25d7c,
3878                 0x25e00, 0x25e04,
3879                 0x26000, 0x2602c,
3880                 0x26100, 0x2613c,
3881                 0x26190, 0x261c8,
3882                 0x26200, 0x26318,
3883                 0x26400, 0x26528,
3884                 0x26540, 0x26614,
3885                 0x27000, 0x27040,
3886                 0x2704c, 0x27060,
3887                 0x270c0, 0x270ec,
3888                 0x27200, 0x27268,
3889                 0x27270, 0x27284,
3890                 0x272fc, 0x27388,
3891                 0x27400, 0x27404,
3892                 0x27500, 0x27518,
3893                 0x2752c, 0x2753c,
3894                 0x27550, 0x27554,
3895                 0x27600, 0x27600,
3896                 0x27608, 0x27628,
3897                 0x27630, 0x2763c,
3898                 0x27700, 0x2771c,
3899                 0x27780, 0x2778c,
3900                 0x27800, 0x27c38,
3901                 0x27c80, 0x27d7c,
3902                 0x27e00, 0x27e04
3903         };
3904         static const unsigned int t5_reg_ranges[] = {
3905                 0x1008, 0x1148,
3906                 0x1180, 0x11b4,
3907                 0x11fc, 0x123c,
3908                 0x1280, 0x173c,
3909                 0x1800, 0x18fc,
3910                 0x3000, 0x3028,
3911                 0x3060, 0x30d8,
3912                 0x30e0, 0x30fc,
3913                 0x3140, 0x357c,
3914                 0x35a8, 0x35cc,
3915                 0x35ec, 0x35ec,
3916                 0x3600, 0x5624,
3917                 0x56cc, 0x575c,
3918                 0x580c, 0x5814,
3919                 0x5890, 0x58bc,
3920                 0x5940, 0x59dc,
3921                 0x59fc, 0x5a18,
3922                 0x5a60, 0x5a9c,
3923                 0x5b94, 0x5bfc,
3924                 0x6000, 0x6040,
3925                 0x6058, 0x614c,
3926                 0x7700, 0x7798,
3927                 0x77c0, 0x78fc,
3928                 0x7b00, 0x7c54,
3929                 0x7d00, 0x7efc,
3930                 0x8dc0, 0x8de0,
3931                 0x8df8, 0x8e84,
3932                 0x8ea0, 0x8f84,
3933                 0x8fc0, 0x90f8,
3934                 0x9400, 0x9470,
3935                 0x9600, 0x96f4,
3936                 0x9800, 0x9808,
3937                 0x9820, 0x983c,
3938                 0x9850, 0x9864,
3939                 0x9c00, 0x9c6c,
3940                 0x9c80, 0x9cec,
3941                 0x9d00, 0x9d6c,
3942                 0x9d80, 0x9dec,
3943                 0x9e00, 0x9e6c,
3944                 0x9e80, 0x9eec,
3945                 0x9f00, 0x9f6c,
3946                 0x9f80, 0xa020,
3947                 0xd004, 0xd03c,
3948                 0xdfc0, 0xdfe0,
3949                 0xe000, 0x11088,
3950                 0x1109c, 0x11110,
3951                 0x11118, 0x1117c,
3952                 0x11190, 0x11204,
3953                 0x19040, 0x1906c,
3954                 0x19078, 0x19080,
3955                 0x1908c, 0x19124,
3956                 0x19150, 0x191b0,
3957                 0x191d0, 0x191e8,
3958                 0x19238, 0x19290,
3959                 0x193f8, 0x19474,
3960                 0x19490, 0x194cc,
3961                 0x194f0, 0x194f8,
3962                 0x19c00, 0x19c60,
3963                 0x19c94, 0x19e10,
3964                 0x19e50, 0x19f34,
3965                 0x19f40, 0x19f50,
3966                 0x19f90, 0x19fe4,
3967                 0x1a000, 0x1a06c,
3968                 0x1a0b0, 0x1a120,
3969                 0x1a128, 0x1a138,
3970                 0x1a190, 0x1a1c4,
3971                 0x1a1fc, 0x1a1fc,
3972                 0x1e008, 0x1e00c,
3973                 0x1e040, 0x1e04c,
3974                 0x1e284, 0x1e290,
3975                 0x1e2c0, 0x1e2c0,
3976                 0x1e2e0, 0x1e2e0,
3977                 0x1e300, 0x1e384,
3978                 0x1e3c0, 0x1e3c8,
3979                 0x1e408, 0x1e40c,
3980                 0x1e440, 0x1e44c,
3981                 0x1e684, 0x1e690,
3982                 0x1e6c0, 0x1e6c0,
3983                 0x1e6e0, 0x1e6e0,
3984                 0x1e700, 0x1e784,
3985                 0x1e7c0, 0x1e7c8,
3986                 0x1e808, 0x1e80c,
3987                 0x1e840, 0x1e84c,
3988                 0x1ea84, 0x1ea90,
3989                 0x1eac0, 0x1eac0,
3990                 0x1eae0, 0x1eae0,
3991                 0x1eb00, 0x1eb84,
3992                 0x1ebc0, 0x1ebc8,
3993                 0x1ec08, 0x1ec0c,
3994                 0x1ec40, 0x1ec4c,
3995                 0x1ee84, 0x1ee90,
3996                 0x1eec0, 0x1eec0,
3997                 0x1eee0, 0x1eee0,
3998                 0x1ef00, 0x1ef84,
3999                 0x1efc0, 0x1efc8,
4000                 0x1f008, 0x1f00c,
4001                 0x1f040, 0x1f04c,
4002                 0x1f284, 0x1f290,
4003                 0x1f2c0, 0x1f2c0,
4004                 0x1f2e0, 0x1f2e0,
4005                 0x1f300, 0x1f384,
4006                 0x1f3c0, 0x1f3c8,
4007                 0x1f408, 0x1f40c,
4008                 0x1f440, 0x1f44c,
4009                 0x1f684, 0x1f690,
4010                 0x1f6c0, 0x1f6c0,
4011                 0x1f6e0, 0x1f6e0,
4012                 0x1f700, 0x1f784,
4013                 0x1f7c0, 0x1f7c8,
4014                 0x1f808, 0x1f80c,
4015                 0x1f840, 0x1f84c,
4016                 0x1fa84, 0x1fa90,
4017                 0x1fac0, 0x1fac0,
4018                 0x1fae0, 0x1fae0,
4019                 0x1fb00, 0x1fb84,
4020                 0x1fbc0, 0x1fbc8,
4021                 0x1fc08, 0x1fc0c,
4022                 0x1fc40, 0x1fc4c,
4023                 0x1fe84, 0x1fe90,
4024                 0x1fec0, 0x1fec0,
4025                 0x1fee0, 0x1fee0,
4026                 0x1ff00, 0x1ff84,
4027                 0x1ffc0, 0x1ffc8,
4028                 0x30000, 0x30030,
4029                 0x30100, 0x30144,
4030                 0x30190, 0x301d0,
4031                 0x30200, 0x30318,
4032                 0x30400, 0x3052c,
4033                 0x30540, 0x3061c,
4034                 0x30800, 0x30834,
4035                 0x308c0, 0x30908,
4036                 0x30910, 0x309ac,
4037                 0x30a00, 0x30a2c,
4038                 0x30a44, 0x30a50,
4039                 0x30a74, 0x30c24,
4040                 0x30d00, 0x30d00,
4041                 0x30d08, 0x30d14,
4042                 0x30d1c, 0x30d20,
4043                 0x30d3c, 0x30d50,
4044                 0x31200, 0x3120c,
4045                 0x31220, 0x31220,
4046                 0x31240, 0x31240,
4047                 0x31600, 0x3160c,
4048                 0x31a00, 0x31a1c,
4049                 0x31e00, 0x31e20,
4050                 0x31e38, 0x31e3c,
4051                 0x31e80, 0x31e80,
4052                 0x31e88, 0x31ea8,
4053                 0x31eb0, 0x31eb4,
4054                 0x31ec8, 0x31ed4,
4055                 0x31fb8, 0x32004,
4056                 0x32200, 0x32200,
4057                 0x32208, 0x32240,
4058                 0x32248, 0x32280,
4059                 0x32288, 0x322c0,
4060                 0x322c8, 0x322fc,
4061                 0x32600, 0x32630,
4062                 0x32a00, 0x32abc,
4063                 0x32b00, 0x32b70,
4064                 0x33000, 0x33048,
4065                 0x33060, 0x3309c,
4066                 0x330f0, 0x33148,
4067                 0x33160, 0x3319c,
4068                 0x331f0, 0x332e4,
4069                 0x332f8, 0x333e4,
4070                 0x333f8, 0x33448,
4071                 0x33460, 0x3349c,
4072                 0x334f0, 0x33548,
4073                 0x33560, 0x3359c,
4074                 0x335f0, 0x336e4,
4075                 0x336f8, 0x337e4,
4076                 0x337f8, 0x337fc,
4077                 0x33814, 0x33814,
4078                 0x3382c, 0x3382c,
4079                 0x33880, 0x3388c,
4080                 0x338e8, 0x338ec,
4081                 0x33900, 0x33948,
4082                 0x33960, 0x3399c,
4083                 0x339f0, 0x33ae4,
4084                 0x33af8, 0x33b10,
4085                 0x33b28, 0x33b28,
4086                 0x33b3c, 0x33b50,
4087                 0x33bf0, 0x33c10,
4088                 0x33c28, 0x33c28,
4089                 0x33c3c, 0x33c50,
4090                 0x33cf0, 0x33cfc,
4091                 0x34000, 0x34030,
4092                 0x34100, 0x34144,
4093                 0x34190, 0x341d0,
4094                 0x34200, 0x34318,
4095                 0x34400, 0x3452c,
4096                 0x34540, 0x3461c,
4097                 0x34800, 0x34834,
4098                 0x348c0, 0x34908,
4099                 0x34910, 0x349ac,
4100                 0x34a00, 0x34a2c,
4101                 0x34a44, 0x34a50,
4102                 0x34a74, 0x34c24,
4103                 0x34d00, 0x34d00,
4104                 0x34d08, 0x34d14,
4105                 0x34d1c, 0x34d20,
4106                 0x34d3c, 0x34d50,
4107                 0x35200, 0x3520c,
4108                 0x35220, 0x35220,
4109                 0x35240, 0x35240,
4110                 0x35600, 0x3560c,
4111                 0x35a00, 0x35a1c,
4112                 0x35e00, 0x35e20,
4113                 0x35e38, 0x35e3c,
4114                 0x35e80, 0x35e80,
4115                 0x35e88, 0x35ea8,
4116                 0x35eb0, 0x35eb4,
4117                 0x35ec8, 0x35ed4,
4118                 0x35fb8, 0x36004,
4119                 0x36200, 0x36200,
4120                 0x36208, 0x36240,
4121                 0x36248, 0x36280,
4122                 0x36288, 0x362c0,
4123                 0x362c8, 0x362fc,
4124                 0x36600, 0x36630,
4125                 0x36a00, 0x36abc,
4126                 0x36b00, 0x36b70,
4127                 0x37000, 0x37048,
4128                 0x37060, 0x3709c,
4129                 0x370f0, 0x37148,
4130                 0x37160, 0x3719c,
4131                 0x371f0, 0x372e4,
4132                 0x372f8, 0x373e4,
4133                 0x373f8, 0x37448,
4134                 0x37460, 0x3749c,
4135                 0x374f0, 0x37548,
4136                 0x37560, 0x3759c,
4137                 0x375f0, 0x376e4,
4138                 0x376f8, 0x377e4,
4139                 0x377f8, 0x377fc,
4140                 0x37814, 0x37814,
4141                 0x3782c, 0x3782c,
4142                 0x37880, 0x3788c,
4143                 0x378e8, 0x378ec,
4144                 0x37900, 0x37948,
4145                 0x37960, 0x3799c,
4146                 0x379f0, 0x37ae4,
4147                 0x37af8, 0x37b10,
4148                 0x37b28, 0x37b28,
4149                 0x37b3c, 0x37b50,
4150                 0x37bf0, 0x37c10,
4151                 0x37c28, 0x37c28,
4152                 0x37c3c, 0x37c50,
4153                 0x37cf0, 0x37cfc,
4154                 0x38000, 0x38030,
4155                 0x38100, 0x38144,
4156                 0x38190, 0x381d0,
4157                 0x38200, 0x38318,
4158                 0x38400, 0x3852c,
4159                 0x38540, 0x3861c,
4160                 0x38800, 0x38834,
4161                 0x388c0, 0x38908,
4162                 0x38910, 0x389ac,
4163                 0x38a00, 0x38a2c,
4164                 0x38a44, 0x38a50,
4165                 0x38a74, 0x38c24,
4166                 0x38d00, 0x38d00,
4167                 0x38d08, 0x38d14,
4168                 0x38d1c, 0x38d20,
4169                 0x38d3c, 0x38d50,
4170                 0x39200, 0x3920c,
4171                 0x39220, 0x39220,
4172                 0x39240, 0x39240,
4173                 0x39600, 0x3960c,
4174                 0x39a00, 0x39a1c,
4175                 0x39e00, 0x39e20,
4176                 0x39e38, 0x39e3c,
4177                 0x39e80, 0x39e80,
4178                 0x39e88, 0x39ea8,
4179                 0x39eb0, 0x39eb4,
4180                 0x39ec8, 0x39ed4,
4181                 0x39fb8, 0x3a004,
4182                 0x3a200, 0x3a200,
4183                 0x3a208, 0x3a240,
4184                 0x3a248, 0x3a280,
4185                 0x3a288, 0x3a2c0,
4186                 0x3a2c8, 0x3a2fc,
4187                 0x3a600, 0x3a630,
4188                 0x3aa00, 0x3aabc,
4189                 0x3ab00, 0x3ab70,
4190                 0x3b000, 0x3b048,
4191                 0x3b060, 0x3b09c,
4192                 0x3b0f0, 0x3b148,
4193                 0x3b160, 0x3b19c,
4194                 0x3b1f0, 0x3b2e4,
4195                 0x3b2f8, 0x3b3e4,
4196                 0x3b3f8, 0x3b448,
4197                 0x3b460, 0x3b49c,
4198                 0x3b4f0, 0x3b548,
4199                 0x3b560, 0x3b59c,
4200                 0x3b5f0, 0x3b6e4,
4201                 0x3b6f8, 0x3b7e4,
4202                 0x3b7f8, 0x3b7fc,
4203                 0x3b814, 0x3b814,
4204                 0x3b82c, 0x3b82c,
4205                 0x3b880, 0x3b88c,
4206                 0x3b8e8, 0x3b8ec,
4207                 0x3b900, 0x3b948,
4208                 0x3b960, 0x3b99c,
4209                 0x3b9f0, 0x3bae4,
4210                 0x3baf8, 0x3bb10,
4211                 0x3bb28, 0x3bb28,
4212                 0x3bb3c, 0x3bb50,
4213                 0x3bbf0, 0x3bc10,
4214                 0x3bc28, 0x3bc28,
4215                 0x3bc3c, 0x3bc50,
4216                 0x3bcf0, 0x3bcfc,
4217                 0x3c000, 0x3c030,
4218                 0x3c100, 0x3c144,
4219                 0x3c190, 0x3c1d0,
4220                 0x3c200, 0x3c318,
4221                 0x3c400, 0x3c52c,
4222                 0x3c540, 0x3c61c,
4223                 0x3c800, 0x3c834,
4224                 0x3c8c0, 0x3c908,
4225                 0x3c910, 0x3c9ac,
4226                 0x3ca00, 0x3ca2c,
4227                 0x3ca44, 0x3ca50,
4228                 0x3ca74, 0x3cc24,
4229                 0x3cd00, 0x3cd00,
4230                 0x3cd08, 0x3cd14,
4231                 0x3cd1c, 0x3cd20,
4232                 0x3cd3c, 0x3cd50,
4233                 0x3d200, 0x3d20c,
4234                 0x3d220, 0x3d220,
4235                 0x3d240, 0x3d240,
4236                 0x3d600, 0x3d60c,
4237                 0x3da00, 0x3da1c,
4238                 0x3de00, 0x3de20,
4239                 0x3de38, 0x3de3c,
4240                 0x3de80, 0x3de80,
4241                 0x3de88, 0x3dea8,
4242                 0x3deb0, 0x3deb4,
4243                 0x3dec8, 0x3ded4,
4244                 0x3dfb8, 0x3e004,
4245                 0x3e200, 0x3e200,
4246                 0x3e208, 0x3e240,
4247                 0x3e248, 0x3e280,
4248                 0x3e288, 0x3e2c0,
4249                 0x3e2c8, 0x3e2fc,
4250                 0x3e600, 0x3e630,
4251                 0x3ea00, 0x3eabc,
4252                 0x3eb00, 0x3eb70,
4253                 0x3f000, 0x3f048,
4254                 0x3f060, 0x3f09c,
4255                 0x3f0f0, 0x3f148,
4256                 0x3f160, 0x3f19c,
4257                 0x3f1f0, 0x3f2e4,
4258                 0x3f2f8, 0x3f3e4,
4259                 0x3f3f8, 0x3f448,
4260                 0x3f460, 0x3f49c,
4261                 0x3f4f0, 0x3f548,
4262                 0x3f560, 0x3f59c,
4263                 0x3f5f0, 0x3f6e4,
4264                 0x3f6f8, 0x3f7e4,
4265                 0x3f7f8, 0x3f7fc,
4266                 0x3f814, 0x3f814,
4267                 0x3f82c, 0x3f82c,
4268                 0x3f880, 0x3f88c,
4269                 0x3f8e8, 0x3f8ec,
4270                 0x3f900, 0x3f948,
4271                 0x3f960, 0x3f99c,
4272                 0x3f9f0, 0x3fae4,
4273                 0x3faf8, 0x3fb10,
4274                 0x3fb28, 0x3fb28,
4275                 0x3fb3c, 0x3fb50,
4276                 0x3fbf0, 0x3fc10,
4277                 0x3fc28, 0x3fc28,
4278                 0x3fc3c, 0x3fc50,
4279                 0x3fcf0, 0x3fcfc,
4280                 0x40000, 0x4000c,
4281                 0x40040, 0x40068,
4282                 0x4007c, 0x40144,
4283                 0x40180, 0x4018c,
4284                 0x40200, 0x40298,
4285                 0x402ac, 0x4033c,
4286                 0x403f8, 0x403fc,
4287                 0x41304, 0x413c4,
4288                 0x41400, 0x4141c,
4289                 0x41480, 0x414d0,
4290                 0x44000, 0x44078,
4291                 0x440c0, 0x44278,
4292                 0x442c0, 0x44478,
4293                 0x444c0, 0x44678,
4294                 0x446c0, 0x44878,
4295                 0x448c0, 0x449fc,
4296                 0x45000, 0x45068,
4297                 0x45080, 0x45084,
4298                 0x450a0, 0x450b0,
4299                 0x45200, 0x45268,
4300                 0x45280, 0x45284,
4301                 0x452a0, 0x452b0,
4302                 0x460c0, 0x460e4,
4303                 0x47000, 0x4708c,
4304                 0x47200, 0x47250,
4305                 0x47400, 0x47420,
4306                 0x47600, 0x47618,
4307                 0x47800, 0x47814,
4308                 0x48000, 0x4800c,
4309                 0x48040, 0x48068,
4310                 0x4807c, 0x48144,
4311                 0x48180, 0x4818c,
4312                 0x48200, 0x48298,
4313                 0x482ac, 0x4833c,
4314                 0x483f8, 0x483fc,
4315                 0x49304, 0x493c4,
4316                 0x49400, 0x4941c,
4317                 0x49480, 0x494d0,
4318                 0x4c000, 0x4c078,
4319                 0x4c0c0, 0x4c278,
4320                 0x4c2c0, 0x4c478,
4321                 0x4c4c0, 0x4c678,
4322                 0x4c6c0, 0x4c878,
4323                 0x4c8c0, 0x4c9fc,
4324                 0x4d000, 0x4d068,
4325                 0x4d080, 0x4d084,
4326                 0x4d0a0, 0x4d0b0,
4327                 0x4d200, 0x4d268,
4328                 0x4d280, 0x4d284,
4329                 0x4d2a0, 0x4d2b0,
4330                 0x4e0c0, 0x4e0e4,
4331                 0x4f000, 0x4f08c,
4332                 0x4f200, 0x4f250,
4333                 0x4f400, 0x4f420,
4334                 0x4f600, 0x4f618,
4335                 0x4f800, 0x4f814,
4336                 0x50000, 0x500cc,
4337                 0x50400, 0x50400,
4338                 0x50800, 0x508cc,
4339                 0x50c00, 0x50c00,
4340                 0x51000, 0x5101c,
4341                 0x51300, 0x51308,
4342         };
4343
4344         if (is_t4(sc)) {
4345                 reg_ranges = &t4_reg_ranges[0];
4346                 n = nitems(t4_reg_ranges);
4347         } else {
4348                 reg_ranges = &t5_reg_ranges[0];
4349                 n = nitems(t5_reg_ranges);
4350         }
4351
4352         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4353         for (i = 0; i < n; i += 2)
4354                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4355 }
4356
4357 static void
4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4359 {
4360         int i;
4361         u_int v, tnl_cong_drops;
4362         struct timeval tv;
4363         const struct timeval interval = {0, 250000};    /* 250ms */
4364
4365         getmicrotime(&tv);
4366         timevalsub(&tv, &interval);
4367         if (timevalcmp(&tv, &pi->last_refreshed, <))
4368                 return;
4369
4370         tnl_cong_drops = 0;
4371         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4372         for (i = 0; i < NCHAN; i++) {
4373                 if (pi->rx_chan_map & (1 << i)) {
4374                         mtx_lock(&sc->regwin_lock);
4375                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4376                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4377                         mtx_unlock(&sc->regwin_lock);
4378                         tnl_cong_drops += v;
4379                 }
4380         }
4381         pi->tnl_cong_drops = tnl_cong_drops;
4382         getmicrotime(&pi->last_refreshed);
4383 }
4384
4385 static void
4386 cxgbe_tick(void *arg)
4387 {
4388         struct port_info *pi = arg;
4389         struct adapter *sc = pi->adapter;
4390         struct ifnet *ifp = pi->ifp;
4391
4392         PORT_LOCK(pi);
4393         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4394                 PORT_UNLOCK(pi);
4395                 return; /* without scheduling another callout */
4396         }
4397
4398         cxgbe_refresh_stats(sc, pi);
4399
4400         callout_schedule(&pi->tick, hz);
4401         PORT_UNLOCK(pi);
4402 }
4403
4404 static void
4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4406 {
4407         struct ifnet *vlan;
4408
4409         if (arg != ifp || ifp->if_type != IFT_ETHER)
4410                 return;
4411
4412         vlan = VLAN_DEVAT(ifp, vid);
4413         VLAN_SETCOOKIE(vlan, ifp);
4414 }
4415
4416 static int
4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4418 {
4419
4420 #ifdef INVARIANTS
4421         panic("%s: opcode 0x%02x on iq %p with payload %p",
4422             __func__, rss->opcode, iq, m);
4423 #else
4424         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4425             __func__, rss->opcode, iq, m);
4426         m_freem(m);
4427 #endif
4428         return (EDOOFUS);
4429 }
4430
4431 int
4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4433 {
4434         uintptr_t *loc, new;
4435
4436         if (opcode >= nitems(sc->cpl_handler))
4437                 return (EINVAL);
4438
4439         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4440         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4441         atomic_store_rel_ptr(loc, new);
4442
4443         return (0);
4444 }
4445
4446 static int
4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4448 {
4449
4450 #ifdef INVARIANTS
4451         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4452 #else
4453         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4454             __func__, iq, ctrl);
4455 #endif
4456         return (EDOOFUS);
4457 }
4458
4459 int
4460 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4461 {
4462         uintptr_t *loc, new;
4463
4464         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4465         loc = (uintptr_t *) &sc->an_handler;
4466         atomic_store_rel_ptr(loc, new);
4467
4468         return (0);
4469 }
4470
4471 static int
4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4473 {
4474         const struct cpl_fw6_msg *cpl =
4475             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4476
4477 #ifdef INVARIANTS
4478         panic("%s: fw_msg type %d", __func__, cpl->type);
4479 #else
4480         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4481 #endif
4482         return (EDOOFUS);
4483 }
4484
4485 int
4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4487 {
4488         uintptr_t *loc, new;
4489
4490         if (type >= nitems(sc->fw_msg_handler))
4491                 return (EINVAL);
4492
4493         /*
4494          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4495          * handler dispatch table.  Reject any attempt to install a handler for
4496          * this subtype.
4497          */
4498         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4499                 return (EINVAL);
4500
4501         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4502         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4503         atomic_store_rel_ptr(loc, new);
4504
4505         return (0);
4506 }
4507
4508 static int
4509 t4_sysctls(struct adapter *sc)
4510 {
4511         struct sysctl_ctx_list *ctx;
4512         struct sysctl_oid *oid;
4513         struct sysctl_oid_list *children, *c0;
4514         static char *caps[] = {
4515                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4516                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4517                     "\6HASHFILTER\7ETHOFLD",
4518                 "\20\1TOE",                             /* caps[2] toecaps */
4519                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4520                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4521                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4522                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4523                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4524                     "\4PO_INITIAOR\5PO_TARGET"
4525         };
4526         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4527
4528         ctx = device_get_sysctl_ctx(sc->dev);
4529
4530         /*
4531          * dev.t4nex.X.
4532          */
4533         oid = device_get_sysctl_tree(sc->dev);
4534         c0 = children = SYSCTL_CHILDREN(oid);
4535
4536         sc->sc_do_rxcopy = 1;
4537         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4538             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4539
4540         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4541             sc->params.nports, "# of ports");
4542
4543         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4544             NULL, chip_rev(sc), "chip hardware revision");
4545
4546         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4547             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4548
4549         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4550             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4551
4552         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4553             sc->cfcsum, "config file checksum");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4556             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4557             sysctl_bitfield, "A", "available doorbells");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4560             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4561             sysctl_bitfield, "A", "available link capabilities");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4564             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4565             sysctl_bitfield, "A", "available NIC capabilities");
4566
4567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4568             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4569             sysctl_bitfield, "A", "available TCP offload capabilities");
4570
4571         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4572             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4573             sysctl_bitfield, "A", "available RDMA capabilities");
4574
4575         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4576             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4577             sysctl_bitfield, "A", "available iSCSI capabilities");
4578
4579         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4580             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4581             sysctl_bitfield, "A", "available FCoE capabilities");
4582
4583         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4584             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4588             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4589             "interrupt holdoff timer values (us)");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4592             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4593             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4594             "interrupt holdoff packet counter values");
4595
4596         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4597             NULL, sc->tids.nftids, "number of filters");
4598
4599         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4600             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4601             "chip temperature (in Celsius)");
4602
4603         t4_sge_sysctls(sc, ctx, children);
4604
4605         sc->lro_timeout = 100;
4606         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4607             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4608
4609 #ifdef SBUF_DRAIN
4610         /*
4611          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4612          */
4613         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4614             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4615             "logs and miscellaneous information");
4616         children = SYSCTL_CHILDREN(oid);
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_cctrl, "A", "congestion control");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4628             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4632             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4636             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4640             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4644             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4648             sysctl_cim_la, "A", "CIM logic analyzer");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4652             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4653
4654         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4655             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4656             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4657
4658         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4659             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4660             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4661
4662         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4663             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4664             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4665
4666         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4667             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4668             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4669
4670         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4671             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4672             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4673
4674         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4675             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4676             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4677
4678         if (is_t5(sc)) {
4679                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4680                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4681                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4682
4683                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4684                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4685                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4686         }
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_cim_qcfg, "A", "CIM queue configuration");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_cpl_stats, "A", "CPL statistics");
4699
4700         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4701             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4702             sysctl_ddp_stats, "A", "DDP statistics");
4703
4704         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4705             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4706             sysctl_devlog, "A", "firmware's device log");
4707
4708         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4709             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4710             sysctl_fcoe_stats, "A", "FCoE statistics");
4711
4712         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4713             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4714             sysctl_hw_sched, "A", "hardware scheduler ");
4715
4716         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4717             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4718             sysctl_l2t, "A", "hardware L2 table");
4719
4720         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4721             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4722             sysctl_lb_stats, "A", "loopback statistics");
4723
4724         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4725             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4726             sysctl_meminfo, "A", "memory regions");
4727
4728         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4729             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4730             sysctl_mps_tcam, "A", "MPS TCAM entries");
4731
4732         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4733             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4734             sysctl_path_mtus, "A", "path MTUs");
4735
4736         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4737             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4738             sysctl_pm_stats, "A", "PM statistics");
4739
4740         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4741             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4742             sysctl_rdma_stats, "A", "RDMA statistics");
4743
4744         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4745             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4746             sysctl_tcp_stats, "A", "TCP statistics");
4747
4748         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4749             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4750             sysctl_tids, "A", "TID information");
4751
4752         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4753             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4754             sysctl_tp_err_stats, "A", "TP error statistics");
4755
4756         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4757             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4758             sysctl_tp_la, "A", "TP logic analyzer");
4759
4760         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4761             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4762             sysctl_tx_rate, "A", "Tx rate");
4763
4764         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4765             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4766             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4767
4768         if (is_t5(sc)) {
4769                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4770                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4771                     sysctl_wcwr_stats, "A", "write combined work requests");
4772         }
4773 #endif
4774
4775 #ifdef TCP_OFFLOAD
4776         if (is_offload(sc)) {
4777                 /*
4778                  * dev.t4nex.X.toe.
4779                  */
4780                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4781                     NULL, "TOE parameters");
4782                 children = SYSCTL_CHILDREN(oid);
4783
4784                 sc->tt.sndbuf = 256 * 1024;
4785                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4786                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4787
4788                 sc->tt.ddp = 0;
4789                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4790                     &sc->tt.ddp, 0, "DDP allowed");
4791
4792                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4793                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4794                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4795
4796                 sc->tt.ddp_thres =
4797                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4798                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4799                     &sc->tt.ddp_thres, 0, "DDP threshold");
4800
4801                 sc->tt.rx_coalesce = 1;
4802                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4803                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4804         }
4805 #endif
4806
4807
4808         return (0);
4809 }
4810
4811 static int
4812 cxgbe_sysctls(struct port_info *pi)
4813 {
4814         struct sysctl_ctx_list *ctx;
4815         struct sysctl_oid *oid;
4816         struct sysctl_oid_list *children;
4817         struct adapter *sc = pi->adapter;
4818
4819         ctx = device_get_sysctl_ctx(pi->dev);
4820
4821         /*
4822          * dev.cxgbe.X.
4823          */
4824         oid = device_get_sysctl_tree(pi->dev);
4825         children = SYSCTL_CHILDREN(oid);
4826
4827         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4828            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4829         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4830                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4831                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4832                     "PHY temperature (in Celsius)");
4833                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4834                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4835                     "PHY firmware version");
4836         }
4837         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4838             &pi->nrxq, 0, "# of rx queues");
4839         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4840             &pi->ntxq, 0, "# of tx queues");
4841         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4842             &pi->first_rxq, 0, "index of first rx queue");
4843         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4844             &pi->first_txq, 0, "index of first tx queue");
4845         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4846             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4847             "Reserve queue 0 for non-flowid packets");
4848
4849 #ifdef TCP_OFFLOAD
4850         if (is_offload(sc)) {
4851                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4852                     &pi->nofldrxq, 0,
4853                     "# of rx queues for offloaded TCP connections");
4854                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4855                     &pi->nofldtxq, 0,
4856                     "# of tx queues for offloaded TCP connections");
4857                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4858                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4859                     "index of first TOE rx queue");
4860                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4861                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4862                     "index of first TOE tx queue");
4863         }
4864 #endif
4865 #ifdef DEV_NETMAP
4866         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4867             &pi->nnmrxq, 0, "# of rx queues for netmap");
4868         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4869             &pi->nnmtxq, 0, "# of tx queues for netmap");
4870         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4871             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4872             "index of first netmap rx queue");
4873         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4874             CTLFLAG_RD, &pi->first_nm_txq, 0,
4875             "index of first netmap tx queue");
4876 #endif
4877
4878         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4879             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4880             "holdoff timer index");
4881         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4882             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4883             "holdoff packet counter index");
4884
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4886             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4887             "rx queue size");
4888         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4889             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4890             "tx queue size");
4891
4892         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4893             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4894             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4895
4896         /*
4897          * dev.cxgbe.X.stats.
4898          */
4899         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4900             NULL, "port statistics");
4901         children = SYSCTL_CHILDREN(oid);
4902         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4903             &pi->tx_parse_error, 0,
4904             "# of tx packets with invalid length or # of segments");
4905
4906 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4907         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4908             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4909             sysctl_handle_t4_reg64, "QU", desc)
4910
4911         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4913         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4924             "# of tx frames in this range",
4925             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4926         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4927             "# of tx frames in this range",
4928             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4929         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4930             "# of tx frames in this range",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4932         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4933             "# of tx frames in this range",
4934             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4935         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4936             "# of tx frames in this range",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4938         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4939             "# of tx frames in this range",
4940             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4941         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4942             "# of tx frames in this range",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4944         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4946         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4954         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4956         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4958         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4960         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4962         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4964
4965         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4966             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4967         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4969         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4972             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4973         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4980             "# of frames received with bad FCS",
4981             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4982         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4983             "# of frames received with length error",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4990             "# of rx frames in this range",
4991             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4992         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4993             "# of rx frames in this range",
4994             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4995         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4996             "# of rx frames in this range",
4997             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4998         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4999             "# of rx frames in this range",
5000             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5001         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5002             "# of rx frames in this range",
5003             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5004         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5005             "# of rx frames in this range",
5006             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5007         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5008             "# of rx frames in this range",
5009             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5010         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5011             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5012         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5018         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5019             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5020         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5021             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5022         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5023             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5024         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5025             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5026         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5027             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5028
5029 #undef SYSCTL_ADD_T4_REG64
5030
5031 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5032         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5033             &pi->stats.name, desc)
5034
5035         /* We get these from port_stats and they may be stale by upto 1s */
5036         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5037             "# drops due to buffer-group 0 overflows");
5038         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5039             "# drops due to buffer-group 1 overflows");
5040         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5041             "# drops due to buffer-group 2 overflows");
5042         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5043             "# drops due to buffer-group 3 overflows");
5044         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5045             "# of buffer-group 0 truncated packets");
5046         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5047             "# of buffer-group 1 truncated packets");
5048         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5049             "# of buffer-group 2 truncated packets");
5050         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5051             "# of buffer-group 3 truncated packets");
5052
5053 #undef SYSCTL_ADD_T4_PORTSTAT
5054
5055         return (0);
5056 }
5057
5058 static int
5059 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5060 {
5061         int rc, *i;
5062         struct sbuf sb;
5063
5064         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5065         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
5066                 sbuf_printf(&sb, "%d ", *i);
5067         sbuf_trim(&sb);
5068         sbuf_finish(&sb);
5069         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5070         sbuf_delete(&sb);
5071         return (rc);
5072 }
5073
5074 static int
5075 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5076 {
5077         int rc;
5078         struct sbuf *sb;
5079
5080         rc = sysctl_wire_old_buffer(req, 0);
5081         if (rc != 0)
5082                 return(rc);
5083
5084         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5085         if (sb == NULL)
5086                 return (ENOMEM);
5087
5088         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5089         rc = sbuf_finish(sb);
5090         sbuf_delete(sb);
5091
5092         return (rc);
5093 }
5094
5095 static int
5096 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5097 {
5098         struct port_info *pi = arg1;
5099         int op = arg2;
5100         struct adapter *sc = pi->adapter;
5101         u_int v;
5102         int rc;
5103
5104         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5105         if (rc)
5106                 return (rc);
5107         /* XXX: magic numbers */
5108         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5109             &v);
5110         end_synchronized_op(sc, 0);
5111         if (rc)
5112                 return (rc);
5113         if (op == 0)
5114                 v /= 256;
5115
5116         rc = sysctl_handle_int(oidp, &v, 0, req);
5117         return (rc);
5118 }
5119
5120 static int
5121 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5122 {
5123         struct port_info *pi = arg1;
5124         int rc, val;
5125
5126         val = pi->rsrv_noflowq;
5127         rc = sysctl_handle_int(oidp, &val, 0, req);
5128         if (rc != 0 || req->newptr == NULL)
5129                 return (rc);
5130
5131         if ((val >= 1) && (pi->ntxq > 1))
5132                 pi->rsrv_noflowq = 1;
5133         else
5134                 pi->rsrv_noflowq = 0;
5135
5136         return (rc);
5137 }
5138
5139 static int
5140 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5141 {
5142         struct port_info *pi = arg1;
5143         struct adapter *sc = pi->adapter;
5144         int idx, rc, i;
5145         struct sge_rxq *rxq;
5146 #ifdef TCP_OFFLOAD
5147         struct sge_ofld_rxq *ofld_rxq;
5148 #endif
5149         uint8_t v;
5150
5151         idx = pi->tmr_idx;
5152
5153         rc = sysctl_handle_int(oidp, &idx, 0, req);
5154         if (rc != 0 || req->newptr == NULL)
5155                 return (rc);
5156
5157         if (idx < 0 || idx >= SGE_NTIMERS)
5158                 return (EINVAL);
5159
5160         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5161             "t4tmr");
5162         if (rc)
5163                 return (rc);
5164
5165         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5166         for_each_rxq(pi, i, rxq) {
5167 #ifdef atomic_store_rel_8
5168                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5169 #else
5170                 rxq->iq.intr_params = v;
5171 #endif
5172         }
5173 #ifdef TCP_OFFLOAD
5174         for_each_ofld_rxq(pi, i, ofld_rxq) {
5175 #ifdef atomic_store_rel_8
5176                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5177 #else
5178                 ofld_rxq->iq.intr_params = v;
5179 #endif
5180         }
5181 #endif
5182         pi->tmr_idx = idx;
5183
5184         end_synchronized_op(sc, LOCK_HELD);
5185         return (0);
5186 }
5187
5188 static int
5189 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5190 {
5191         struct port_info *pi = arg1;
5192         struct adapter *sc = pi->adapter;
5193         int idx, rc;
5194
5195         idx = pi->pktc_idx;
5196
5197         rc = sysctl_handle_int(oidp, &idx, 0, req);
5198         if (rc != 0 || req->newptr == NULL)
5199                 return (rc);
5200
5201         if (idx < -1 || idx >= SGE_NCOUNTERS)
5202                 return (EINVAL);
5203
5204         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5205             "t4pktc");
5206         if (rc)
5207                 return (rc);
5208
5209         if (pi->flags & PORT_INIT_DONE)
5210                 rc = EBUSY; /* cannot be changed once the queues are created */
5211         else
5212                 pi->pktc_idx = idx;
5213
5214         end_synchronized_op(sc, LOCK_HELD);
5215         return (rc);
5216 }
5217
5218 static int
5219 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5220 {
5221         struct port_info *pi = arg1;
5222         struct adapter *sc = pi->adapter;
5223         int qsize, rc;
5224
5225         qsize = pi->qsize_rxq;
5226
5227         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5228         if (rc != 0 || req->newptr == NULL)
5229                 return (rc);
5230
5231         if (qsize < 128 || (qsize & 7))
5232                 return (EINVAL);
5233
5234         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5235             "t4rxqs");
5236         if (rc)
5237                 return (rc);
5238
5239         if (pi->flags & PORT_INIT_DONE)
5240                 rc = EBUSY; /* cannot be changed once the queues are created */
5241         else
5242                 pi->qsize_rxq = qsize;
5243
5244         end_synchronized_op(sc, LOCK_HELD);
5245         return (rc);
5246 }
5247
5248 static int
5249 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5250 {
5251         struct port_info *pi = arg1;
5252         struct adapter *sc = pi->adapter;
5253         int qsize, rc;
5254
5255         qsize = pi->qsize_txq;
5256
5257         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5258         if (rc != 0 || req->newptr == NULL)
5259                 return (rc);
5260
5261         /* bufring size must be powerof2 */
5262         if (qsize < 128 || !powerof2(qsize))
5263                 return (EINVAL);
5264
5265         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5266             "t4txqs");
5267         if (rc)
5268                 return (rc);
5269
5270         if (pi->flags & PORT_INIT_DONE)
5271                 rc = EBUSY; /* cannot be changed once the queues are created */
5272         else
5273                 pi->qsize_txq = qsize;
5274
5275         end_synchronized_op(sc, LOCK_HELD);
5276         return (rc);
5277 }
5278
5279 static int
5280 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5281 {
5282         struct port_info *pi = arg1;
5283         struct adapter *sc = pi->adapter;
5284         struct link_config *lc = &pi->link_cfg;
5285         int rc;
5286
5287         if (req->newptr == NULL) {
5288                 struct sbuf *sb;
5289                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5290
5291                 rc = sysctl_wire_old_buffer(req, 0);
5292                 if (rc != 0)
5293                         return(rc);
5294
5295                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5296                 if (sb == NULL)
5297                         return (ENOMEM);
5298
5299                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5300                 rc = sbuf_finish(sb);
5301                 sbuf_delete(sb);
5302         } else {
5303                 char s[2];
5304                 int n;
5305
5306                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5307                 s[1] = 0;
5308
5309                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5310                 if (rc != 0)
5311                         return(rc);
5312
5313                 if (s[1] != 0)
5314                         return (EINVAL);
5315                 if (s[0] < '0' || s[0] > '9')
5316                         return (EINVAL);        /* not a number */
5317                 n = s[0] - '0';
5318                 if (n & ~(PAUSE_TX | PAUSE_RX))
5319                         return (EINVAL);        /* some other bit is set too */
5320
5321                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5322                 if (rc)
5323                         return (rc);
5324                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5325                         int link_ok = lc->link_ok;
5326
5327                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5328                         lc->requested_fc |= n;
5329                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5330                         lc->link_ok = link_ok;  /* restore */
5331                 }
5332                 end_synchronized_op(sc, 0);
5333         }
5334
5335         return (rc);
5336 }
5337
5338 static int
5339 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5340 {
5341         struct adapter *sc = arg1;
5342         int reg = arg2;
5343         uint64_t val;
5344
5345         val = t4_read_reg64(sc, reg);
5346
5347         return (sysctl_handle_64(oidp, &val, 0, req));
5348 }
5349
5350 static int
5351 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5352 {
5353         struct adapter *sc = arg1;
5354         int rc, t;
5355         uint32_t param, val;
5356
5357         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5358         if (rc)
5359                 return (rc);
5360         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5361             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5362             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5363         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5364         end_synchronized_op(sc, 0);
5365         if (rc)
5366                 return (rc);
5367
5368         /* unknown is returned as 0 but we display -1 in that case */
5369         t = val == 0 ? -1 : val;
5370
5371         rc = sysctl_handle_int(oidp, &t, 0, req);
5372         return (rc);
5373 }
5374
5375 #ifdef SBUF_DRAIN
5376 static int
5377 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5378 {
5379         struct adapter *sc = arg1;
5380         struct sbuf *sb;
5381         int rc, i;
5382         uint16_t incr[NMTUS][NCCTRL_WIN];
5383         static const char *dec_fac[] = {
5384                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5385                 "0.9375"
5386         };
5387
5388         rc = sysctl_wire_old_buffer(req, 0);
5389         if (rc != 0)
5390                 return (rc);
5391
5392         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5393         if (sb == NULL)
5394                 return (ENOMEM);
5395
5396         t4_read_cong_tbl(sc, incr);
5397
5398         for (i = 0; i < NCCTRL_WIN; ++i) {
5399                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5400                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5401                     incr[5][i], incr[6][i], incr[7][i]);
5402                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5403                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5404                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5405                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5406         }
5407
5408         rc = sbuf_finish(sb);
5409         sbuf_delete(sb);
5410
5411         return (rc);
5412 }
5413
5414 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5415         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5416         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5417         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5418 };
5419
5420 static int
5421 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5422 {
5423         struct adapter *sc = arg1;
5424         struct sbuf *sb;
5425         int rc, i, n, qid = arg2;
5426         uint32_t *buf, *p;
5427         char *qtype;
5428         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5429
5430         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5431             ("%s: bad qid %d\n", __func__, qid));
5432
5433         if (qid < CIM_NUM_IBQ) {
5434                 /* inbound queue */
5435                 qtype = "IBQ";
5436                 n = 4 * CIM_IBQ_SIZE;
5437                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5438                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5439         } else {
5440                 /* outbound queue */
5441                 qtype = "OBQ";
5442                 qid -= CIM_NUM_IBQ;
5443                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5444                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5445                 rc = t4_read_cim_obq(sc, qid, buf, n);
5446         }
5447
5448         if (rc < 0) {
5449                 rc = -rc;
5450                 goto done;
5451         }
5452         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5453
5454         rc = sysctl_wire_old_buffer(req, 0);
5455         if (rc != 0)
5456                 goto done;
5457
5458         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5459         if (sb == NULL) {
5460                 rc = ENOMEM;
5461                 goto done;
5462         }
5463
5464         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5465         for (i = 0, p = buf; i < n; i += 16, p += 4)
5466                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5467                     p[2], p[3]);
5468
5469         rc = sbuf_finish(sb);
5470         sbuf_delete(sb);
5471 done:
5472         free(buf, M_CXGBE);
5473         return (rc);
5474 }
5475
5476 static int
5477 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5478 {
5479         struct adapter *sc = arg1;
5480         u_int cfg;
5481         struct sbuf *sb;
5482         uint32_t *buf, *p;
5483         int rc;
5484
5485         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5486         if (rc != 0)
5487                 return (rc);
5488
5489         rc = sysctl_wire_old_buffer(req, 0);
5490         if (rc != 0)
5491                 return (rc);
5492
5493         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5494         if (sb == NULL)
5495                 return (ENOMEM);
5496
5497         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5498             M_ZERO | M_WAITOK);
5499
5500         rc = -t4_cim_read_la(sc, buf, NULL);
5501         if (rc != 0)
5502                 goto done;
5503
5504         sbuf_printf(sb, "Status   Data      PC%s",
5505             cfg & F_UPDBGLACAPTPCONLY ? "" :
5506             "     LS0Stat  LS0Addr             LS0Data");
5507
5508         KASSERT((sc->params.cim_la_size & 7) == 0,
5509             ("%s: p will walk off the end of buf", __func__));
5510
5511         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5512                 if (cfg & F_UPDBGLACAPTPCONLY) {
5513                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5514                             p[6], p[7]);
5515                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5516                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5517                             p[4] & 0xff, p[5] >> 8);
5518                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5519                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5520                             p[1] & 0xf, p[2] >> 4);
5521                 } else {
5522                         sbuf_printf(sb,
5523                             "\n  %02x   %x%07x %x%07x %08x %08x "
5524                             "%08x%08x%08x%08x",
5525                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5526                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5527                             p[6], p[7]);
5528                 }
5529         }
5530
5531         rc = sbuf_finish(sb);
5532         sbuf_delete(sb);
5533 done:
5534         free(buf, M_CXGBE);
5535         return (rc);
5536 }
5537
5538 static int
5539 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5540 {
5541         struct adapter *sc = arg1;
5542         u_int i;
5543         struct sbuf *sb;
5544         uint32_t *buf, *p;
5545         int rc;
5546
5547         rc = sysctl_wire_old_buffer(req, 0);
5548         if (rc != 0)
5549                 return (rc);
5550
5551         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5552         if (sb == NULL)
5553                 return (ENOMEM);
5554
5555         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5556             M_ZERO | M_WAITOK);
5557
5558         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5559         p = buf;
5560
5561         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5562                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5563                     p[1], p[0]);
5564         }
5565
5566         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5567         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5568                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5569                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5570                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5571                     (p[1] >> 2) | ((p[2] & 3) << 30),
5572                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5573                     p[0] & 1);
5574         }
5575
5576         rc = sbuf_finish(sb);
5577         sbuf_delete(sb);
5578         free(buf, M_CXGBE);
5579         return (rc);
5580 }
5581
5582 static int
5583 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5584 {
5585         struct adapter *sc = arg1;
5586         u_int i;
5587         struct sbuf *sb;
5588         uint32_t *buf, *p;
5589         int rc;
5590
5591         rc = sysctl_wire_old_buffer(req, 0);
5592         if (rc != 0)
5593                 return (rc);
5594
5595         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5596         if (sb == NULL)
5597                 return (ENOMEM);
5598
5599         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5600             M_ZERO | M_WAITOK);
5601
5602         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5603         p = buf;
5604
5605         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5606         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5607                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5608                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5609                     p[4], p[3], p[2], p[1], p[0]);
5610         }
5611
5612         sbuf_printf(sb, "\n\nCntl ID               Data");
5613         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5614                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5615                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5616         }
5617
5618         rc = sbuf_finish(sb);
5619         sbuf_delete(sb);
5620         free(buf, M_CXGBE);
5621         return (rc);
5622 }
5623
5624 static int
5625 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5626 {
5627         struct adapter *sc = arg1;
5628         struct sbuf *sb;
5629         int rc, i;
5630         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5631         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5632         uint16_t thres[CIM_NUM_IBQ];
5633         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5634         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5635         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5636
5637         if (is_t4(sc)) {
5638                 cim_num_obq = CIM_NUM_OBQ;
5639                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5640                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5641         } else {
5642                 cim_num_obq = CIM_NUM_OBQ_T5;
5643                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5644                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5645         }
5646         nq = CIM_NUM_IBQ + cim_num_obq;
5647
5648         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5649         if (rc == 0)
5650                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5651         if (rc != 0)
5652                 return (rc);
5653
5654         t4_read_cimq_cfg(sc, base, size, thres);
5655
5656         rc = sysctl_wire_old_buffer(req, 0);
5657         if (rc != 0)
5658                 return (rc);
5659
5660         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5661         if (sb == NULL)
5662                 return (ENOMEM);
5663
5664         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5665
5666         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5667                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5668                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5669                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5670                     G_QUEREMFLITS(p[2]) * 16);
5671         for ( ; i < nq; i++, p += 4, wr += 2)
5672                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5673                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5674                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5675                     G_QUEREMFLITS(p[2]) * 16);
5676
5677         rc = sbuf_finish(sb);
5678         sbuf_delete(sb);
5679
5680         return (rc);
5681 }
5682
5683 static int
5684 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5685 {
5686         struct adapter *sc = arg1;
5687         struct sbuf *sb;
5688         int rc;
5689         struct tp_cpl_stats stats;
5690
5691         rc = sysctl_wire_old_buffer(req, 0);
5692         if (rc != 0)
5693                 return (rc);
5694
5695         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5696         if (sb == NULL)
5697                 return (ENOMEM);
5698
5699         t4_tp_get_cpl_stats(sc, &stats);
5700
5701         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5702             "channel 3\n");
5703         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5704                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5705         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5706                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5707
5708         rc = sbuf_finish(sb);
5709         sbuf_delete(sb);
5710
5711         return (rc);
5712 }
5713
5714 static int
5715 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5716 {
5717         struct adapter *sc = arg1;
5718         struct sbuf *sb;
5719         int rc;
5720         struct tp_usm_stats stats;
5721
5722         rc = sysctl_wire_old_buffer(req, 0);
5723         if (rc != 0)
5724                 return(rc);
5725
5726         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5727         if (sb == NULL)
5728                 return (ENOMEM);
5729
5730         t4_get_usm_stats(sc, &stats);
5731
5732         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5733         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5734         sbuf_printf(sb, "Drops:  %u", stats.drops);
5735
5736         rc = sbuf_finish(sb);
5737         sbuf_delete(sb);
5738
5739         return (rc);
5740 }
5741
5742 const char *devlog_level_strings[] = {
5743         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5744         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5745         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5746         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5747         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5748         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5749 };
5750
5751 const char *devlog_facility_strings[] = {
5752         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5753         [FW_DEVLOG_FACILITY_CF]         = "CF",
5754         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5755         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5756         [FW_DEVLOG_FACILITY_RES]        = "RES",
5757         [FW_DEVLOG_FACILITY_HW]         = "HW",
5758         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5759         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5760         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5761         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5762         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5763         [FW_DEVLOG_FACILITY_VI]         = "VI",
5764         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5765         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5766         [FW_DEVLOG_FACILITY_TM]         = "TM",
5767         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5768         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5769         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5770         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5771         [FW_DEVLOG_FACILITY_RI]         = "RI",
5772         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5773         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5774         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5775         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5776 };
5777
5778 static int
5779 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5780 {
5781         struct adapter *sc = arg1;
5782         struct devlog_params *dparams = &sc->params.devlog;
5783         struct fw_devlog_e *buf, *e;
5784         int i, j, rc, nentries, first = 0, m;
5785         struct sbuf *sb;
5786         uint64_t ftstamp = UINT64_MAX;
5787
5788         if (dparams->start == 0) {
5789                 dparams->memtype = FW_MEMTYPE_EDC0;
5790                 dparams->start = 0x84000;
5791                 dparams->size = 32768;
5792         }
5793
5794         nentries = dparams->size / sizeof(struct fw_devlog_e);
5795
5796         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5797         if (buf == NULL)
5798                 return (ENOMEM);
5799
5800         m = fwmtype_to_hwmtype(dparams->memtype);
5801         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5802         if (rc != 0)
5803                 goto done;
5804
5805         for (i = 0; i < nentries; i++) {
5806                 e = &buf[i];
5807
5808                 if (e->timestamp == 0)
5809                         break;  /* end */
5810
5811                 e->timestamp = be64toh(e->timestamp);
5812                 e->seqno = be32toh(e->seqno);
5813                 for (j = 0; j < 8; j++)
5814                         e->params[j] = be32toh(e->params[j]);
5815
5816                 if (e->timestamp < ftstamp) {
5817                         ftstamp = e->timestamp;
5818                         first = i;
5819                 }
5820         }
5821
5822         if (buf[first].timestamp == 0)
5823                 goto done;      /* nothing in the log */
5824
5825         rc = sysctl_wire_old_buffer(req, 0);
5826         if (rc != 0)
5827                 goto done;
5828
5829         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5830         if (sb == NULL) {
5831                 rc = ENOMEM;
5832                 goto done;
5833         }
5834         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5835             "Seq#", "Tstamp", "Level", "Facility", "Message");
5836
5837         i = first;
5838         do {
5839                 e = &buf[i];
5840                 if (e->timestamp == 0)
5841                         break;  /* end */
5842
5843                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5844                     e->seqno, e->timestamp,
5845                     (e->level < nitems(devlog_level_strings) ?
5846                         devlog_level_strings[e->level] : "UNKNOWN"),
5847                     (e->facility < nitems(devlog_facility_strings) ?
5848                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5849                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5850                     e->params[2], e->params[3], e->params[4],
5851                     e->params[5], e->params[6], e->params[7]);
5852
5853                 if (++i == nentries)
5854                         i = 0;
5855         } while (i != first);
5856
5857         rc = sbuf_finish(sb);
5858         sbuf_delete(sb);
5859 done:
5860         free(buf, M_CXGBE);
5861         return (rc);
5862 }
5863
5864 static int
5865 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5866 {
5867         struct adapter *sc = arg1;
5868         struct sbuf *sb;
5869         int rc;
5870         struct tp_fcoe_stats stats[4];
5871
5872         rc = sysctl_wire_old_buffer(req, 0);
5873         if (rc != 0)
5874                 return (rc);
5875
5876         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5877         if (sb == NULL)
5878                 return (ENOMEM);
5879
5880         t4_get_fcoe_stats(sc, 0, &stats[0]);
5881         t4_get_fcoe_stats(sc, 1, &stats[1]);
5882         t4_get_fcoe_stats(sc, 2, &stats[2]);
5883         t4_get_fcoe_stats(sc, 3, &stats[3]);
5884
5885         sbuf_printf(sb, "                   channel 0        channel 1        "
5886             "channel 2        channel 3\n");
5887         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5888             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5889             stats[3].octetsDDP);
5890         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5891             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5892         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5893             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5894             stats[3].framesDrop);
5895
5896         rc = sbuf_finish(sb);
5897         sbuf_delete(sb);
5898
5899         return (rc);
5900 }
5901
5902 static int
5903 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5904 {
5905         struct adapter *sc = arg1;
5906         struct sbuf *sb;
5907         int rc, i;
5908         unsigned int map, kbps, ipg, mode;
5909         unsigned int pace_tab[NTX_SCHED];
5910
5911         rc = sysctl_wire_old_buffer(req, 0);
5912         if (rc != 0)
5913                 return (rc);
5914
5915         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5916         if (sb == NULL)
5917                 return (ENOMEM);
5918
5919         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5920         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5921         t4_read_pace_tbl(sc, pace_tab);
5922
5923         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5924             "Class IPG (0.1 ns)   Flow IPG (us)");
5925
5926         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5927                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5928                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5929                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5930                 if (kbps)
5931                         sbuf_printf(sb, "%9u     ", kbps);
5932                 else
5933                         sbuf_printf(sb, " disabled     ");
5934
5935                 if (ipg)
5936                         sbuf_printf(sb, "%13u        ", ipg);
5937                 else
5938                         sbuf_printf(sb, "     disabled        ");
5939
5940                 if (pace_tab[i])
5941                         sbuf_printf(sb, "%10u", pace_tab[i]);
5942                 else
5943                         sbuf_printf(sb, "  disabled");
5944         }
5945
5946         rc = sbuf_finish(sb);
5947         sbuf_delete(sb);
5948
5949         return (rc);
5950 }
5951
5952 static int
5953 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5954 {
5955         struct adapter *sc = arg1;
5956         struct sbuf *sb;
5957         int rc, i, j;
5958         uint64_t *p0, *p1;
5959         struct lb_port_stats s[2];
5960         static const char *stat_name[] = {
5961                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5962                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5963                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5964                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5965                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5966                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5967                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5968         };
5969
5970         rc = sysctl_wire_old_buffer(req, 0);
5971         if (rc != 0)
5972                 return (rc);
5973
5974         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5975         if (sb == NULL)
5976                 return (ENOMEM);
5977
5978         memset(s, 0, sizeof(s));
5979
5980         for (i = 0; i < 4; i += 2) {
5981                 t4_get_lb_stats(sc, i, &s[0]);
5982                 t4_get_lb_stats(sc, i + 1, &s[1]);
5983
5984                 p0 = &s[0].octets;
5985                 p1 = &s[1].octets;
5986                 sbuf_printf(sb, "%s                       Loopback %u"
5987                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5988
5989                 for (j = 0; j < nitems(stat_name); j++)
5990                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5991                                    *p0++, *p1++);
5992         }
5993
5994         rc = sbuf_finish(sb);
5995         sbuf_delete(sb);
5996
5997         return (rc);
5998 }
5999
6000 static int
6001 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6002 {
6003         int rc = 0;
6004         struct port_info *pi = arg1;
6005         struct sbuf *sb;
6006         static const char *linkdnreasons[] = {
6007                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6008                 "PHY overheated", "unknown", "rx los", "reserved7"
6009         };
6010
6011         rc = sysctl_wire_old_buffer(req, 0);
6012         if (rc != 0)
6013                 return(rc);
6014         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6015         if (sb == NULL)
6016                 return (ENOMEM);
6017
6018         if (pi->linkdnrc < 0)
6019                 sbuf_printf(sb, "n/a");
6020         else if (pi->linkdnrc < nitems(linkdnreasons))
6021                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6022         else
6023                 sbuf_printf(sb, "%d", pi->linkdnrc);
6024
6025         rc = sbuf_finish(sb);
6026         sbuf_delete(sb);
6027
6028         return (rc);
6029 }
6030
6031 struct mem_desc {
6032         unsigned int base;
6033         unsigned int limit;
6034         unsigned int idx;
6035 };
6036
6037 static int
6038 mem_desc_cmp(const void *a, const void *b)
6039 {
6040         return ((const struct mem_desc *)a)->base -
6041                ((const struct mem_desc *)b)->base;
6042 }
6043
6044 static void
6045 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6046     unsigned int to)
6047 {
6048         unsigned int size;
6049
6050         size = to - from + 1;
6051         if (size == 0)
6052                 return;
6053
6054         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6055         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6056 }
6057
6058 static int
6059 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6060 {
6061         struct adapter *sc = arg1;
6062         struct sbuf *sb;
6063         int rc, i, n;
6064         uint32_t lo, hi, used, alloc;
6065         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6066         static const char *region[] = {
6067                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6068                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6069                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6070                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6071                 "RQUDP region:", "PBL region:", "TXPBL region:",
6072                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6073                 "On-chip queues:"
6074         };
6075         struct mem_desc avail[4];
6076         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6077         struct mem_desc *md = mem;
6078
6079         rc = sysctl_wire_old_buffer(req, 0);
6080         if (rc != 0)
6081                 return (rc);
6082
6083         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6084         if (sb == NULL)
6085                 return (ENOMEM);
6086
6087         for (i = 0; i < nitems(mem); i++) {
6088                 mem[i].limit = 0;
6089                 mem[i].idx = i;
6090         }
6091
6092         /* Find and sort the populated memory ranges */
6093         i = 0;
6094         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6095         if (lo & F_EDRAM0_ENABLE) {
6096                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6097                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6098                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6099                 avail[i].idx = 0;
6100                 i++;
6101         }
6102         if (lo & F_EDRAM1_ENABLE) {
6103                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6104                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6105                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6106                 avail[i].idx = 1;
6107                 i++;
6108         }
6109         if (lo & F_EXT_MEM_ENABLE) {
6110                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6111                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6112                 avail[i].limit = avail[i].base +
6113                     (G_EXT_MEM_SIZE(hi) << 20);
6114                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6115                 i++;
6116         }
6117         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6118                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6119                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6120                 avail[i].limit = avail[i].base +
6121                     (G_EXT_MEM1_SIZE(hi) << 20);
6122                 avail[i].idx = 4;
6123                 i++;
6124         }
6125         if (!i)                                    /* no memory available */
6126                 return 0;
6127         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6128
6129         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6130         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6131         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6132         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6133         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6134         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6135         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6136         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6137         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6138
6139         /* the next few have explicit upper bounds */
6140         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6141         md->limit = md->base - 1 +
6142                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6143                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6144         md++;
6145
6146         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6147         md->limit = md->base - 1 +
6148                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6149                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6150         md++;
6151
6152         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6153                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6154                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6155                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6156         } else {
6157                 md->base = 0;
6158                 md->idx = nitems(region);  /* hide it */
6159         }
6160         md++;
6161
6162 #define ulp_region(reg) \
6163         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6164         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6165
6166         ulp_region(RX_ISCSI);
6167         ulp_region(RX_TDDP);
6168         ulp_region(TX_TPT);
6169         ulp_region(RX_STAG);
6170         ulp_region(RX_RQ);
6171         ulp_region(RX_RQUDP);
6172         ulp_region(RX_PBL);
6173         ulp_region(TX_PBL);
6174 #undef ulp_region
6175
6176         md->base = 0;
6177         md->idx = nitems(region);
6178         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6179                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6180                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6181                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6182         }
6183         md++;
6184
6185         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6186         md->limit = md->base + sc->tids.ntids - 1;
6187         md++;
6188         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6189         md->limit = md->base + sc->tids.ntids - 1;
6190         md++;
6191
6192         md->base = sc->vres.ocq.start;
6193         if (sc->vres.ocq.size)
6194                 md->limit = md->base + sc->vres.ocq.size - 1;
6195         else
6196                 md->idx = nitems(region);  /* hide it */
6197         md++;
6198
6199         /* add any address-space holes, there can be up to 3 */
6200         for (n = 0; n < i - 1; n++)
6201                 if (avail[n].limit < avail[n + 1].base)
6202                         (md++)->base = avail[n].limit;
6203         if (avail[n].limit)
6204                 (md++)->base = avail[n].limit;
6205
6206         n = md - mem;
6207         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6208
6209         for (lo = 0; lo < i; lo++)
6210                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6211                                 avail[lo].limit - 1);
6212
6213         sbuf_printf(sb, "\n");
6214         for (i = 0; i < n; i++) {
6215                 if (mem[i].idx >= nitems(region))
6216                         continue;                        /* skip holes */
6217                 if (!mem[i].limit)
6218                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6219                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6220                                 mem[i].limit);
6221         }
6222
6223         sbuf_printf(sb, "\n");
6224         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6225         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6226         mem_region_show(sb, "uP RAM:", lo, hi);
6227
6228         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6229         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6230         mem_region_show(sb, "uP Extmem2:", lo, hi);
6231
6232         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6233         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6234                    G_PMRXMAXPAGE(lo),
6235                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6236                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6237
6238         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6239         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6240         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6241                    G_PMTXMAXPAGE(lo),
6242                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6243                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6244         sbuf_printf(sb, "%u p-structs\n",
6245                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6246
6247         for (i = 0; i < 4; i++) {
6248                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6249                 if (is_t4(sc)) {
6250                         used = G_USED(lo);
6251                         alloc = G_ALLOC(lo);
6252                 } else {
6253                         used = G_T5_USED(lo);
6254                         alloc = G_T5_ALLOC(lo);
6255                 }
6256                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6257                            i, used, alloc);
6258         }
6259         for (i = 0; i < 4; i++) {
6260                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6261                 if (is_t4(sc)) {
6262                         used = G_USED(lo);
6263                         alloc = G_ALLOC(lo);
6264                 } else {
6265                         used = G_T5_USED(lo);
6266                         alloc = G_T5_ALLOC(lo);
6267                 }
6268                 sbuf_printf(sb,
6269                            "\nLoopback %d using %u pages out of %u allocated",
6270                            i, used, alloc);
6271         }
6272
6273         rc = sbuf_finish(sb);
6274         sbuf_delete(sb);
6275
6276         return (rc);
6277 }
6278
6279 static inline void
6280 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6281 {
6282         *mask = x | y;
6283         y = htobe64(y);
6284         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6285 }
6286
6287 static int
6288 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6289 {
6290         struct adapter *sc = arg1;
6291         struct sbuf *sb;
6292         int rc, i, n;
6293
6294         rc = sysctl_wire_old_buffer(req, 0);
6295         if (rc != 0)
6296                 return (rc);
6297
6298         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6299         if (sb == NULL)
6300                 return (ENOMEM);
6301
6302         sbuf_printf(sb,
6303             "Idx  Ethernet address     Mask     Vld Ports PF"
6304             "  VF              Replication             P0 P1 P2 P3  ML");
6305         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6306             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6307         for (i = 0; i < n; i++) {
6308                 uint64_t tcamx, tcamy, mask;
6309                 uint32_t cls_lo, cls_hi;
6310                 uint8_t addr[ETHER_ADDR_LEN];
6311
6312                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6313                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6314                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6315                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6316
6317                 if (tcamx & tcamy)
6318                         continue;
6319
6320                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6321                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6322                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6323                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6324                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6325                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6326                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6327
6328                 if (cls_lo & F_REPLICATE) {
6329                         struct fw_ldst_cmd ldst_cmd;
6330
6331                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6332                         ldst_cmd.op_to_addrspace =
6333                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6334                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6335                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6336                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6337                         ldst_cmd.u.mps.fid_ctl =
6338                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6339                                 V_FW_LDST_CMD_CTL(i));
6340
6341                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6342                             "t4mps");
6343                         if (rc)
6344                                 break;
6345                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6346                             sizeof(ldst_cmd), &ldst_cmd);
6347                         end_synchronized_op(sc, 0);
6348
6349                         if (rc != 0) {
6350                                 sbuf_printf(sb,
6351                                     " ------------ error %3u ------------", rc);
6352                                 rc = 0;
6353                         } else {
6354                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6355                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6356                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6357                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6358                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6359                         }
6360                 } else
6361                         sbuf_printf(sb, "%36s", "");
6362
6363                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6364                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6365                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6366         }
6367
6368         if (rc)
6369                 (void) sbuf_finish(sb);
6370         else
6371                 rc = sbuf_finish(sb);
6372         sbuf_delete(sb);
6373
6374         return (rc);
6375 }
6376
6377 static int
6378 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6379 {
6380         struct adapter *sc = arg1;
6381         struct sbuf *sb;
6382         int rc;
6383         uint16_t mtus[NMTUS];
6384
6385         rc = sysctl_wire_old_buffer(req, 0);
6386         if (rc != 0)
6387                 return (rc);
6388
6389         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6390         if (sb == NULL)
6391                 return (ENOMEM);
6392
6393         t4_read_mtu_tbl(sc, mtus, NULL);
6394
6395         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6396             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6397             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6398             mtus[14], mtus[15]);
6399
6400         rc = sbuf_finish(sb);
6401         sbuf_delete(sb);
6402
6403         return (rc);
6404 }
6405
6406 static int
6407 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6408 {
6409         struct adapter *sc = arg1;
6410         struct sbuf *sb;
6411         int rc, i;
6412         uint32_t cnt[PM_NSTATS];
6413         uint64_t cyc[PM_NSTATS];
6414         static const char *rx_stats[] = {
6415                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6416         };
6417         static const char *tx_stats[] = {
6418                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6419         };
6420
6421         rc = sysctl_wire_old_buffer(req, 0);
6422         if (rc != 0)
6423                 return (rc);
6424
6425         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6426         if (sb == NULL)
6427                 return (ENOMEM);
6428
6429         t4_pmtx_get_stats(sc, cnt, cyc);
6430         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6431         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6432                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6433                     cyc[i]);
6434
6435         t4_pmrx_get_stats(sc, cnt, cyc);
6436         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6437         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6438                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6439                     cyc[i]);
6440
6441         rc = sbuf_finish(sb);
6442         sbuf_delete(sb);
6443
6444         return (rc);
6445 }
6446
6447 static int
6448 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6449 {
6450         struct adapter *sc = arg1;
6451         struct sbuf *sb;
6452         int rc;
6453         struct tp_rdma_stats stats;
6454
6455         rc = sysctl_wire_old_buffer(req, 0);
6456         if (rc != 0)
6457                 return (rc);
6458
6459         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6460         if (sb == NULL)
6461                 return (ENOMEM);
6462
6463         t4_tp_get_rdma_stats(sc, &stats);
6464         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6465         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6466
6467         rc = sbuf_finish(sb);
6468         sbuf_delete(sb);
6469
6470         return (rc);
6471 }
6472
6473 static int
6474 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6475 {
6476         struct adapter *sc = arg1;
6477         struct sbuf *sb;
6478         int rc;
6479         struct tp_tcp_stats v4, v6;
6480
6481         rc = sysctl_wire_old_buffer(req, 0);
6482         if (rc != 0)
6483                 return (rc);
6484
6485         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6486         if (sb == NULL)
6487                 return (ENOMEM);
6488
6489         t4_tp_get_tcp_stats(sc, &v4, &v6);
6490         sbuf_printf(sb,
6491             "                                IP                 IPv6\n");
6492         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6493             v4.tcpOutRsts, v6.tcpOutRsts);
6494         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6495             v4.tcpInSegs, v6.tcpInSegs);
6496         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6497             v4.tcpOutSegs, v6.tcpOutSegs);
6498         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6499             v4.tcpRetransSegs, v6.tcpRetransSegs);
6500
6501         rc = sbuf_finish(sb);
6502         sbuf_delete(sb);
6503
6504         return (rc);
6505 }
6506
6507 static int
6508 sysctl_tids(SYSCTL_HANDLER_ARGS)
6509 {
6510         struct adapter *sc = arg1;
6511         struct sbuf *sb;
6512         int rc;
6513         struct tid_info *t = &sc->tids;
6514
6515         rc = sysctl_wire_old_buffer(req, 0);
6516         if (rc != 0)
6517                 return (rc);
6518
6519         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6520         if (sb == NULL)
6521                 return (ENOMEM);
6522
6523         if (t->natids) {
6524                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6525                     t->atids_in_use);
6526         }
6527
6528         if (t->ntids) {
6529                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6530                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6531
6532                         if (b) {
6533                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6534                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6535                                     t->ntids - 1);
6536                         } else {
6537                                 sbuf_printf(sb, "TID range: %u-%u",
6538                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6539                                     t->ntids - 1);
6540                         }
6541                 } else
6542                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6543                 sbuf_printf(sb, ", in use: %u\n",
6544                     atomic_load_acq_int(&t->tids_in_use));
6545         }
6546
6547         if (t->nstids) {
6548                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6549                     t->stid_base + t->nstids - 1, t->stids_in_use);
6550         }
6551
6552         if (t->nftids) {
6553                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6554                     t->ftid_base + t->nftids - 1);
6555         }
6556
6557         if (t->netids) {
6558                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6559                     t->etid_base + t->netids - 1);
6560         }
6561
6562         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6563             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6564             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6565
6566         rc = sbuf_finish(sb);
6567         sbuf_delete(sb);
6568
6569         return (rc);
6570 }
6571
6572 static int
6573 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6574 {
6575         struct adapter *sc = arg1;
6576         struct sbuf *sb;
6577         int rc;
6578         struct tp_err_stats stats;
6579
6580         rc = sysctl_wire_old_buffer(req, 0);
6581         if (rc != 0)
6582                 return (rc);
6583
6584         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6585         if (sb == NULL)
6586                 return (ENOMEM);
6587
6588         t4_tp_get_err_stats(sc, &stats);
6589
6590         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6591                       "channel 3\n");
6592         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6593             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6594             stats.macInErrs[3]);
6595         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6596             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6597             stats.hdrInErrs[3]);
6598         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6599             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6600             stats.tcpInErrs[3]);
6601         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6602             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6603             stats.tcp6InErrs[3]);
6604         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6605             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6606             stats.tnlCongDrops[3]);
6607         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6608             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6609             stats.tnlTxDrops[3]);
6610         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6611             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6612             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6613         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6614             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6615             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6616         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6617             stats.ofldNoNeigh, stats.ofldCongDefer);
6618
6619         rc = sbuf_finish(sb);
6620         sbuf_delete(sb);
6621
6622         return (rc);
6623 }
6624
6625 struct field_desc {
6626         const char *name;
6627         u_int start;
6628         u_int width;
6629 };
6630
6631 static void
6632 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6633 {
6634         char buf[32];
6635         int line_size = 0;
6636
6637         while (f->name) {
6638                 uint64_t mask = (1ULL << f->width) - 1;
6639                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6640                     ((uintmax_t)v >> f->start) & mask);
6641
6642                 if (line_size + len >= 79) {
6643                         line_size = 8;
6644                         sbuf_printf(sb, "\n        ");
6645                 }
6646                 sbuf_printf(sb, "%s ", buf);
6647                 line_size += len + 1;
6648                 f++;
6649         }
6650         sbuf_printf(sb, "\n");
6651 }
6652
6653 static struct field_desc tp_la0[] = {
6654         { "RcfOpCodeOut", 60, 4 },
6655         { "State", 56, 4 },
6656         { "WcfState", 52, 4 },
6657         { "RcfOpcSrcOut", 50, 2 },
6658         { "CRxError", 49, 1 },
6659         { "ERxError", 48, 1 },
6660         { "SanityFailed", 47, 1 },
6661         { "SpuriousMsg", 46, 1 },
6662         { "FlushInputMsg", 45, 1 },
6663         { "FlushInputCpl", 44, 1 },
6664         { "RssUpBit", 43, 1 },
6665         { "RssFilterHit", 42, 1 },
6666         { "Tid", 32, 10 },
6667         { "InitTcb", 31, 1 },
6668         { "LineNumber", 24, 7 },
6669         { "Emsg", 23, 1 },
6670         { "EdataOut", 22, 1 },
6671         { "Cmsg", 21, 1 },
6672         { "CdataOut", 20, 1 },
6673         { "EreadPdu", 19, 1 },
6674         { "CreadPdu", 18, 1 },
6675         { "TunnelPkt", 17, 1 },
6676         { "RcfPeerFin", 16, 1 },
6677         { "RcfReasonOut", 12, 4 },
6678         { "TxCchannel", 10, 2 },
6679         { "RcfTxChannel", 8, 2 },
6680         { "RxEchannel", 6, 2 },
6681         { "RcfRxChannel", 5, 1 },
6682         { "RcfDataOutSrdy", 4, 1 },
6683         { "RxDvld", 3, 1 },
6684         { "RxOoDvld", 2, 1 },
6685         { "RxCongestion", 1, 1 },
6686         { "TxCongestion", 0, 1 },
6687         { NULL }
6688 };
6689
6690 static struct field_desc tp_la1[] = {
6691         { "CplCmdIn", 56, 8 },
6692         { "CplCmdOut", 48, 8 },
6693         { "ESynOut", 47, 1 },
6694         { "EAckOut", 46, 1 },
6695         { "EFinOut", 45, 1 },
6696         { "ERstOut", 44, 1 },
6697         { "SynIn", 43, 1 },
6698         { "AckIn", 42, 1 },
6699         { "FinIn", 41, 1 },
6700         { "RstIn", 40, 1 },
6701         { "DataIn", 39, 1 },
6702         { "DataInVld", 38, 1 },
6703         { "PadIn", 37, 1 },
6704         { "RxBufEmpty", 36, 1 },
6705         { "RxDdp", 35, 1 },
6706         { "RxFbCongestion", 34, 1 },
6707         { "TxFbCongestion", 33, 1 },
6708         { "TxPktSumSrdy", 32, 1 },
6709         { "RcfUlpType", 28, 4 },
6710         { "Eread", 27, 1 },
6711         { "Ebypass", 26, 1 },
6712         { "Esave", 25, 1 },
6713         { "Static0", 24, 1 },
6714         { "Cread", 23, 1 },
6715         { "Cbypass", 22, 1 },
6716         { "Csave", 21, 1 },
6717         { "CPktOut", 20, 1 },
6718         { "RxPagePoolFull", 18, 2 },
6719         { "RxLpbkPkt", 17, 1 },
6720         { "TxLpbkPkt", 16, 1 },
6721         { "RxVfValid", 15, 1 },
6722         { "SynLearned", 14, 1 },
6723         { "SetDelEntry", 13, 1 },
6724         { "SetInvEntry", 12, 1 },
6725         { "CpcmdDvld", 11, 1 },
6726         { "CpcmdSave", 10, 1 },
6727         { "RxPstructsFull", 8, 2 },
6728         { "EpcmdDvld", 7, 1 },
6729         { "EpcmdFlush", 6, 1 },
6730         { "EpcmdTrimPrefix", 5, 1 },
6731         { "EpcmdTrimPostfix", 4, 1 },
6732         { "ERssIp4Pkt", 3, 1 },
6733         { "ERssIp6Pkt", 2, 1 },
6734         { "ERssTcpUdpPkt", 1, 1 },
6735         { "ERssFceFipPkt", 0, 1 },
6736         { NULL }
6737 };
6738
6739 static struct field_desc tp_la2[] = {
6740         { "CplCmdIn", 56, 8 },
6741         { "MpsVfVld", 55, 1 },
6742         { "MpsPf", 52, 3 },
6743         { "MpsVf", 44, 8 },
6744         { "SynIn", 43, 1 },
6745         { "AckIn", 42, 1 },
6746         { "FinIn", 41, 1 },
6747         { "RstIn", 40, 1 },
6748         { "DataIn", 39, 1 },
6749         { "DataInVld", 38, 1 },
6750         { "PadIn", 37, 1 },
6751         { "RxBufEmpty", 36, 1 },
6752         { "RxDdp", 35, 1 },
6753         { "RxFbCongestion", 34, 1 },
6754         { "TxFbCongestion", 33, 1 },
6755         { "TxPktSumSrdy", 32, 1 },
6756         { "RcfUlpType", 28, 4 },
6757         { "Eread", 27, 1 },
6758         { "Ebypass", 26, 1 },
6759         { "Esave", 25, 1 },
6760         { "Static0", 24, 1 },
6761         { "Cread", 23, 1 },
6762         { "Cbypass", 22, 1 },
6763         { "Csave", 21, 1 },
6764         { "CPktOut", 20, 1 },
6765         { "RxPagePoolFull", 18, 2 },
6766         { "RxLpbkPkt", 17, 1 },
6767         { "TxLpbkPkt", 16, 1 },
6768         { "RxVfValid", 15, 1 },
6769         { "SynLearned", 14, 1 },
6770         { "SetDelEntry", 13, 1 },
6771         { "SetInvEntry", 12, 1 },
6772         { "CpcmdDvld", 11, 1 },
6773         { "CpcmdSave", 10, 1 },
6774         { "RxPstructsFull", 8, 2 },
6775         { "EpcmdDvld", 7, 1 },
6776         { "EpcmdFlush", 6, 1 },
6777         { "EpcmdTrimPrefix", 5, 1 },
6778         { "EpcmdTrimPostfix", 4, 1 },
6779         { "ERssIp4Pkt", 3, 1 },
6780         { "ERssIp6Pkt", 2, 1 },
6781         { "ERssTcpUdpPkt", 1, 1 },
6782         { "ERssFceFipPkt", 0, 1 },
6783         { NULL }
6784 };
6785
6786 static void
6787 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6788 {
6789
6790         field_desc_show(sb, *p, tp_la0);
6791 }
6792
6793 static void
6794 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6795 {
6796
6797         if (idx)
6798                 sbuf_printf(sb, "\n");
6799         field_desc_show(sb, p[0], tp_la0);
6800         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6801                 field_desc_show(sb, p[1], tp_la0);
6802 }
6803
6804 static void
6805 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6806 {
6807
6808         if (idx)
6809                 sbuf_printf(sb, "\n");
6810         field_desc_show(sb, p[0], tp_la0);
6811         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6812                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6813 }
6814
6815 static int
6816 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6817 {
6818         struct adapter *sc = arg1;
6819         struct sbuf *sb;
6820         uint64_t *buf, *p;
6821         int rc;
6822         u_int i, inc;
6823         void (*show_func)(struct sbuf *, uint64_t *, int);
6824
6825         rc = sysctl_wire_old_buffer(req, 0);
6826         if (rc != 0)
6827                 return (rc);
6828
6829         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6830         if (sb == NULL)
6831                 return (ENOMEM);
6832
6833         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6834
6835         t4_tp_read_la(sc, buf, NULL);
6836         p = buf;
6837
6838         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6839         case 2:
6840                 inc = 2;
6841                 show_func = tp_la_show2;
6842                 break;
6843         case 3:
6844                 inc = 2;
6845                 show_func = tp_la_show3;
6846                 break;
6847         default:
6848                 inc = 1;
6849                 show_func = tp_la_show;
6850         }
6851
6852         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6853                 (*show_func)(sb, p, i);
6854
6855         rc = sbuf_finish(sb);
6856         sbuf_delete(sb);
6857         free(buf, M_CXGBE);
6858         return (rc);
6859 }
6860
6861 static int
6862 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6863 {
6864         struct adapter *sc = arg1;
6865         struct sbuf *sb;
6866         int rc;
6867         u64 nrate[NCHAN], orate[NCHAN];
6868
6869         rc = sysctl_wire_old_buffer(req, 0);
6870         if (rc != 0)
6871                 return (rc);
6872
6873         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6874         if (sb == NULL)
6875                 return (ENOMEM);
6876
6877         t4_get_chan_txrate(sc, nrate, orate);
6878         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6879                  "channel 3\n");
6880         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6881             nrate[0], nrate[1], nrate[2], nrate[3]);
6882         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6883             orate[0], orate[1], orate[2], orate[3]);
6884
6885         rc = sbuf_finish(sb);
6886         sbuf_delete(sb);
6887
6888         return (rc);
6889 }
6890
6891 static int
6892 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6893 {
6894         struct adapter *sc = arg1;
6895         struct sbuf *sb;
6896         uint32_t *buf, *p;
6897         int rc, i;
6898
6899         rc = sysctl_wire_old_buffer(req, 0);
6900         if (rc != 0)
6901                 return (rc);
6902
6903         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6904         if (sb == NULL)
6905                 return (ENOMEM);
6906
6907         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6908             M_ZERO | M_WAITOK);
6909
6910         t4_ulprx_read_la(sc, buf);
6911         p = buf;
6912
6913         sbuf_printf(sb, "      Pcmd        Type   Message"
6914             "                Data");
6915         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6916                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6917                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6918         }
6919
6920         rc = sbuf_finish(sb);
6921         sbuf_delete(sb);
6922         free(buf, M_CXGBE);
6923         return (rc);
6924 }
6925
6926 static int
6927 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6928 {
6929         struct adapter *sc = arg1;
6930         struct sbuf *sb;
6931         int rc, v;
6932
6933         rc = sysctl_wire_old_buffer(req, 0);
6934         if (rc != 0)
6935                 return (rc);
6936
6937         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6938         if (sb == NULL)
6939                 return (ENOMEM);
6940
6941         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6942         if (G_STATSOURCE_T5(v) == 7) {
6943                 if (G_STATMODE(v) == 0) {
6944                         sbuf_printf(sb, "total %d, incomplete %d",
6945                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6946                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6947                 } else if (G_STATMODE(v) == 1) {
6948                         sbuf_printf(sb, "total %d, data overflow %d",
6949                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6950                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6951                 }
6952         }
6953         rc = sbuf_finish(sb);
6954         sbuf_delete(sb);
6955
6956         return (rc);
6957 }
6958 #endif
6959
6960 static uint32_t
6961 fconf_to_mode(uint32_t fconf)
6962 {
6963         uint32_t mode;
6964
6965         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6966             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6967
6968         if (fconf & F_FRAGMENTATION)
6969                 mode |= T4_FILTER_IP_FRAGMENT;
6970
6971         if (fconf & F_MPSHITTYPE)
6972                 mode |= T4_FILTER_MPS_HIT_TYPE;
6973
6974         if (fconf & F_MACMATCH)
6975                 mode |= T4_FILTER_MAC_IDX;
6976
6977         if (fconf & F_ETHERTYPE)
6978                 mode |= T4_FILTER_ETH_TYPE;
6979
6980         if (fconf & F_PROTOCOL)
6981                 mode |= T4_FILTER_IP_PROTO;
6982
6983         if (fconf & F_TOS)
6984                 mode |= T4_FILTER_IP_TOS;
6985
6986         if (fconf & F_VLAN)
6987                 mode |= T4_FILTER_VLAN;
6988
6989         if (fconf & F_VNIC_ID)
6990                 mode |= T4_FILTER_VNIC;
6991
6992         if (fconf & F_PORT)
6993                 mode |= T4_FILTER_PORT;
6994
6995         if (fconf & F_FCOE)
6996                 mode |= T4_FILTER_FCoE;
6997
6998         return (mode);
6999 }
7000
7001 static uint32_t
7002 mode_to_fconf(uint32_t mode)
7003 {
7004         uint32_t fconf = 0;
7005
7006         if (mode & T4_FILTER_IP_FRAGMENT)
7007                 fconf |= F_FRAGMENTATION;
7008
7009         if (mode & T4_FILTER_MPS_HIT_TYPE)
7010                 fconf |= F_MPSHITTYPE;
7011
7012         if (mode & T4_FILTER_MAC_IDX)
7013                 fconf |= F_MACMATCH;
7014
7015         if (mode & T4_FILTER_ETH_TYPE)
7016                 fconf |= F_ETHERTYPE;
7017
7018         if (mode & T4_FILTER_IP_PROTO)
7019                 fconf |= F_PROTOCOL;
7020
7021         if (mode & T4_FILTER_IP_TOS)
7022                 fconf |= F_TOS;
7023
7024         if (mode & T4_FILTER_VLAN)
7025                 fconf |= F_VLAN;
7026
7027         if (mode & T4_FILTER_VNIC)
7028                 fconf |= F_VNIC_ID;
7029
7030         if (mode & T4_FILTER_PORT)
7031                 fconf |= F_PORT;
7032
7033         if (mode & T4_FILTER_FCoE)
7034                 fconf |= F_FCOE;
7035
7036         return (fconf);
7037 }
7038
7039 static uint32_t
7040 fspec_to_fconf(struct t4_filter_specification *fs)
7041 {
7042         uint32_t fconf = 0;
7043
7044         if (fs->val.frag || fs->mask.frag)
7045                 fconf |= F_FRAGMENTATION;
7046
7047         if (fs->val.matchtype || fs->mask.matchtype)
7048                 fconf |= F_MPSHITTYPE;
7049
7050         if (fs->val.macidx || fs->mask.macidx)
7051                 fconf |= F_MACMATCH;
7052
7053         if (fs->val.ethtype || fs->mask.ethtype)
7054                 fconf |= F_ETHERTYPE;
7055
7056         if (fs->val.proto || fs->mask.proto)
7057                 fconf |= F_PROTOCOL;
7058
7059         if (fs->val.tos || fs->mask.tos)
7060                 fconf |= F_TOS;
7061
7062         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7063                 fconf |= F_VLAN;
7064
7065         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7066                 fconf |= F_VNIC_ID;
7067
7068         if (fs->val.iport || fs->mask.iport)
7069                 fconf |= F_PORT;
7070
7071         if (fs->val.fcoe || fs->mask.fcoe)
7072                 fconf |= F_FCOE;
7073
7074         return (fconf);
7075 }
7076
7077 static int
7078 get_filter_mode(struct adapter *sc, uint32_t *mode)
7079 {
7080         int rc;
7081         uint32_t fconf;
7082
7083         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7084             "t4getfm");
7085         if (rc)
7086                 return (rc);
7087
7088         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7089             A_TP_VLAN_PRI_MAP);
7090
7091         if (sc->params.tp.vlan_pri_map != fconf) {
7092                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7093                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7094                     fconf);
7095                 sc->params.tp.vlan_pri_map = fconf;
7096         }
7097
7098         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7099
7100         end_synchronized_op(sc, LOCK_HELD);
7101         return (0);
7102 }
7103
7104 static int
7105 set_filter_mode(struct adapter *sc, uint32_t mode)
7106 {
7107         uint32_t fconf;
7108         int rc;
7109
7110         fconf = mode_to_fconf(mode);
7111
7112         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7113             "t4setfm");
7114         if (rc)
7115                 return (rc);
7116
7117         if (sc->tids.ftids_in_use > 0) {
7118                 rc = EBUSY;
7119                 goto done;
7120         }
7121
7122 #ifdef TCP_OFFLOAD
7123         if (sc->offload_map) {
7124                 rc = EBUSY;
7125                 goto done;
7126         }
7127 #endif
7128
7129 #ifdef notyet
7130         rc = -t4_set_filter_mode(sc, fconf);
7131         if (rc == 0)
7132                 sc->filter_mode = fconf;
7133 #else
7134         rc = ENOTSUP;
7135 #endif
7136
7137 done:
7138         end_synchronized_op(sc, LOCK_HELD);
7139         return (rc);
7140 }
7141
7142 static inline uint64_t
7143 get_filter_hits(struct adapter *sc, uint32_t fid)
7144 {
7145         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7146         uint64_t hits;
7147
7148         memwin_info(sc, 0, &mw_base, NULL);
7149         off = position_memwin(sc, 0,
7150             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7151         if (is_t4(sc)) {
7152                 hits = t4_read_reg64(sc, mw_base + off + 16);
7153                 hits = be64toh(hits);
7154         } else {
7155                 hits = t4_read_reg(sc, mw_base + off + 24);
7156                 hits = be32toh(hits);
7157         }
7158
7159         return (hits);
7160 }
7161
7162 static int
7163 get_filter(struct adapter *sc, struct t4_filter *t)
7164 {
7165         int i, rc, nfilters = sc->tids.nftids;
7166         struct filter_entry *f;
7167
7168         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7169             "t4getf");
7170         if (rc)
7171                 return (rc);
7172
7173         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7174             t->idx >= nfilters) {
7175                 t->idx = 0xffffffff;
7176                 goto done;
7177         }
7178
7179         f = &sc->tids.ftid_tab[t->idx];
7180         for (i = t->idx; i < nfilters; i++, f++) {
7181                 if (f->valid) {
7182                         t->idx = i;
7183                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7184                         t->smtidx = f->smtidx;
7185                         if (f->fs.hitcnts)
7186                                 t->hits = get_filter_hits(sc, t->idx);
7187                         else
7188                                 t->hits = UINT64_MAX;
7189                         t->fs = f->fs;
7190
7191                         goto done;
7192                 }
7193         }
7194
7195         t->idx = 0xffffffff;
7196 done:
7197         end_synchronized_op(sc, LOCK_HELD);
7198         return (0);
7199 }
7200
7201 static int
7202 set_filter(struct adapter *sc, struct t4_filter *t)
7203 {
7204         unsigned int nfilters, nports;
7205         struct filter_entry *f;
7206         int i, rc;
7207
7208         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7209         if (rc)
7210                 return (rc);
7211
7212         nfilters = sc->tids.nftids;
7213         nports = sc->params.nports;
7214
7215         if (nfilters == 0) {
7216                 rc = ENOTSUP;
7217                 goto done;
7218         }
7219
7220         if (!(sc->flags & FULL_INIT_DONE)) {
7221                 rc = EAGAIN;
7222                 goto done;
7223         }
7224
7225         if (t->idx >= nfilters) {
7226                 rc = EINVAL;
7227                 goto done;
7228         }
7229
7230         /* Validate against the global filter mode */
7231         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7232             sc->params.tp.vlan_pri_map) {
7233                 rc = E2BIG;
7234                 goto done;
7235         }
7236
7237         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7238                 rc = EINVAL;
7239                 goto done;
7240         }
7241
7242         if (t->fs.val.iport >= nports) {
7243                 rc = EINVAL;
7244                 goto done;
7245         }
7246
7247         /* Can't specify an iq if not steering to it */
7248         if (!t->fs.dirsteer && t->fs.iq) {
7249                 rc = EINVAL;
7250                 goto done;
7251         }
7252
7253         /* IPv6 filter idx must be 4 aligned */
7254         if (t->fs.type == 1 &&
7255             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7256                 rc = EINVAL;
7257                 goto done;
7258         }
7259
7260         if (sc->tids.ftid_tab == NULL) {
7261                 KASSERT(sc->tids.ftids_in_use == 0,
7262                     ("%s: no memory allocated but filters_in_use > 0",
7263                     __func__));
7264
7265                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7266                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7267                 if (sc->tids.ftid_tab == NULL) {
7268                         rc = ENOMEM;
7269                         goto done;
7270                 }
7271                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7272         }
7273
7274         for (i = 0; i < 4; i++) {
7275                 f = &sc->tids.ftid_tab[t->idx + i];
7276
7277                 if (f->pending || f->valid) {
7278                         rc = EBUSY;
7279                         goto done;
7280                 }
7281                 if (f->locked) {
7282                         rc = EPERM;
7283                         goto done;
7284                 }
7285
7286                 if (t->fs.type == 0)
7287                         break;
7288         }
7289
7290         f = &sc->tids.ftid_tab[t->idx];
7291         f->fs = t->fs;
7292
7293         rc = set_filter_wr(sc, t->idx);
7294 done:
7295         end_synchronized_op(sc, 0);
7296
7297         if (rc == 0) {
7298                 mtx_lock(&sc->tids.ftid_lock);
7299                 for (;;) {
7300                         if (f->pending == 0) {
7301                                 rc = f->valid ? 0 : EIO;
7302                                 break;
7303                         }
7304
7305                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7306                             PCATCH, "t4setfw", 0)) {
7307                                 rc = EINPROGRESS;
7308                                 break;
7309                         }
7310                 }
7311                 mtx_unlock(&sc->tids.ftid_lock);
7312         }
7313         return (rc);
7314 }
7315
7316 static int
7317 del_filter(struct adapter *sc, struct t4_filter *t)
7318 {
7319         unsigned int nfilters;
7320         struct filter_entry *f;
7321         int rc;
7322
7323         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7324         if (rc)
7325                 return (rc);
7326
7327         nfilters = sc->tids.nftids;
7328
7329         if (nfilters == 0) {
7330                 rc = ENOTSUP;
7331                 goto done;
7332         }
7333
7334         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7335             t->idx >= nfilters) {
7336                 rc = EINVAL;
7337                 goto done;
7338         }
7339
7340         if (!(sc->flags & FULL_INIT_DONE)) {
7341                 rc = EAGAIN;
7342                 goto done;
7343         }
7344
7345         f = &sc->tids.ftid_tab[t->idx];
7346
7347         if (f->pending) {
7348                 rc = EBUSY;
7349                 goto done;
7350         }
7351         if (f->locked) {
7352                 rc = EPERM;
7353                 goto done;
7354         }
7355
7356         if (f->valid) {
7357                 t->fs = f->fs;  /* extra info for the caller */
7358                 rc = del_filter_wr(sc, t->idx);
7359         }
7360
7361 done:
7362         end_synchronized_op(sc, 0);
7363
7364         if (rc == 0) {
7365                 mtx_lock(&sc->tids.ftid_lock);
7366                 for (;;) {
7367                         if (f->pending == 0) {
7368                                 rc = f->valid ? EIO : 0;
7369                                 break;
7370                         }
7371
7372                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7373                             PCATCH, "t4delfw", 0)) {
7374                                 rc = EINPROGRESS;
7375                                 break;
7376                         }
7377                 }
7378                 mtx_unlock(&sc->tids.ftid_lock);
7379         }
7380
7381         return (rc);
7382 }
7383
7384 static void
7385 clear_filter(struct filter_entry *f)
7386 {
7387         if (f->l2t)
7388                 t4_l2t_release(f->l2t);
7389
7390         bzero(f, sizeof (*f));
7391 }
7392
7393 static int
7394 set_filter_wr(struct adapter *sc, int fidx)
7395 {
7396         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7397         struct fw_filter_wr *fwr;
7398         unsigned int ftid;
7399         struct wrq_cookie cookie;
7400
7401         ASSERT_SYNCHRONIZED_OP(sc);
7402
7403         if (f->fs.newdmac || f->fs.newvlan) {
7404                 /* This filter needs an L2T entry; allocate one. */
7405                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7406                 if (f->l2t == NULL)
7407                         return (EAGAIN);
7408                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7409                     f->fs.dmac)) {
7410                         t4_l2t_release(f->l2t);
7411                         f->l2t = NULL;
7412                         return (ENOMEM);
7413                 }
7414         }
7415
7416         ftid = sc->tids.ftid_base + fidx;
7417
7418         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7419         if (fwr == NULL)
7420                 return (ENOMEM);
7421         bzero(fwr, sizeof(*fwr));
7422
7423         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7424         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7425         fwr->tid_to_iq =
7426             htobe32(V_FW_FILTER_WR_TID(ftid) |
7427                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7428                 V_FW_FILTER_WR_NOREPLY(0) |
7429                 V_FW_FILTER_WR_IQ(f->fs.iq));
7430         fwr->del_filter_to_l2tix =
7431             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7432                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7433                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7434                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7435                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7436                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7437                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7438                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7439                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7440                     f->fs.newvlan == VLAN_REWRITE) |
7441                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7442                     f->fs.newvlan == VLAN_REWRITE) |
7443                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7444                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7445                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7446                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7447         fwr->ethtype = htobe16(f->fs.val.ethtype);
7448         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7449         fwr->frag_to_ovlan_vldm =
7450             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7451                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7452                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7453                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7454                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7455                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7456         fwr->smac_sel = 0;
7457         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7458             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7459         fwr->maci_to_matchtypem =
7460             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7461                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7462                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7463                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7464                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7465                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7466                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7467                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7468         fwr->ptcl = f->fs.val.proto;
7469         fwr->ptclm = f->fs.mask.proto;
7470         fwr->ttyp = f->fs.val.tos;
7471         fwr->ttypm = f->fs.mask.tos;
7472         fwr->ivlan = htobe16(f->fs.val.vlan);
7473         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7474         fwr->ovlan = htobe16(f->fs.val.vnic);
7475         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7476         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7477         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7478         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7479         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7480         fwr->lp = htobe16(f->fs.val.dport);
7481         fwr->lpm = htobe16(f->fs.mask.dport);
7482         fwr->fp = htobe16(f->fs.val.sport);
7483         fwr->fpm = htobe16(f->fs.mask.sport);
7484         if (f->fs.newsmac)
7485                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7486
7487         f->pending = 1;
7488         sc->tids.ftids_in_use++;
7489
7490         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7491         return (0);
7492 }
7493
7494 static int
7495 del_filter_wr(struct adapter *sc, int fidx)
7496 {
7497         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7498         struct fw_filter_wr *fwr;
7499         unsigned int ftid;
7500         struct wrq_cookie cookie;
7501
7502         ftid = sc->tids.ftid_base + fidx;
7503
7504         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7505         if (fwr == NULL)
7506                 return (ENOMEM);
7507         bzero(fwr, sizeof (*fwr));
7508
7509         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7510
7511         f->pending = 1;
7512         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7513         return (0);
7514 }
7515
7516 int
7517 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7518 {
7519         struct adapter *sc = iq->adapter;
7520         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7521         unsigned int idx = GET_TID(rpl);
7522         unsigned int rc;
7523         struct filter_entry *f;
7524
7525         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7526             rss->opcode));
7527
7528         if (is_ftid(sc, idx)) {
7529
7530                 idx -= sc->tids.ftid_base;
7531                 f = &sc->tids.ftid_tab[idx];
7532                 rc = G_COOKIE(rpl->cookie);
7533
7534                 mtx_lock(&sc->tids.ftid_lock);
7535                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7536                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7537                             __func__, idx));
7538                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7539                         f->pending = 0;  /* asynchronous setup completed */
7540                         f->valid = 1;
7541                 } else {
7542                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7543                                 /* Add or delete failed, display an error */
7544                                 log(LOG_ERR,
7545                                     "filter %u setup failed with error %u\n",
7546                                     idx, rc);
7547                         }
7548
7549                         clear_filter(f);
7550                         sc->tids.ftids_in_use--;
7551                 }
7552                 wakeup(&sc->tids.ftid_tab);
7553                 mtx_unlock(&sc->tids.ftid_lock);
7554         }
7555
7556         return (0);
7557 }
7558
7559 static int
7560 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7561 {
7562         int rc;
7563
7564         if (cntxt->cid > M_CTXTQID)
7565                 return (EINVAL);
7566
7567         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7568             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7569                 return (EINVAL);
7570
7571         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7572         if (rc)
7573                 return (rc);
7574
7575         if (sc->flags & FW_OK) {
7576                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7577                     &cntxt->data[0]);
7578                 if (rc == 0)
7579                         goto done;
7580         }
7581
7582         /*
7583          * Read via firmware failed or wasn't even attempted.  Read directly via
7584          * the backdoor.
7585          */
7586         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7587 done:
7588         end_synchronized_op(sc, 0);
7589         return (rc);
7590 }
7591
7592 static int
7593 load_fw(struct adapter *sc, struct t4_data *fw)
7594 {
7595         int rc;
7596         uint8_t *fw_data;
7597
7598         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7599         if (rc)
7600                 return (rc);
7601
7602         if (sc->flags & FULL_INIT_DONE) {
7603                 rc = EBUSY;
7604                 goto done;
7605         }
7606
7607         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7608         if (fw_data == NULL) {
7609                 rc = ENOMEM;
7610                 goto done;
7611         }
7612
7613         rc = copyin(fw->data, fw_data, fw->len);
7614         if (rc == 0)
7615                 rc = -t4_load_fw(sc, fw_data, fw->len);
7616
7617         free(fw_data, M_CXGBE);
7618 done:
7619         end_synchronized_op(sc, 0);
7620         return (rc);
7621 }
7622
7623 static int
7624 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7625 {
7626         uint32_t addr, off, remaining, i, n;
7627         uint32_t *buf, *b;
7628         uint32_t mw_base, mw_aperture;
7629         int rc;
7630         uint8_t *dst;
7631
7632         rc = validate_mem_range(sc, mr->addr, mr->len);
7633         if (rc != 0)
7634                 return (rc);
7635
7636         memwin_info(sc, win, &mw_base, &mw_aperture);
7637         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7638         addr = mr->addr;
7639         remaining = mr->len;
7640         dst = (void *)mr->data;
7641
7642         while (remaining) {
7643                 off = position_memwin(sc, win, addr);
7644
7645                 /* number of bytes that we'll copy in the inner loop */
7646                 n = min(remaining, mw_aperture - off);
7647                 for (i = 0; i < n; i += 4)
7648                         *b++ = t4_read_reg(sc, mw_base + off + i);
7649
7650                 rc = copyout(buf, dst, n);
7651                 if (rc != 0)
7652                         break;
7653
7654                 b = buf;
7655                 dst += n;
7656                 remaining -= n;
7657                 addr += n;
7658         }
7659
7660         free(buf, M_CXGBE);
7661         return (rc);
7662 }
7663
7664 static int
7665 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7666 {
7667         int rc;
7668
7669         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7670                 return (EINVAL);
7671
7672         if (i2cd->len > sizeof(i2cd->data))
7673                 return (EFBIG);
7674
7675         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7676         if (rc)
7677                 return (rc);
7678         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7679             i2cd->offset, i2cd->len, &i2cd->data[0]);
7680         end_synchronized_op(sc, 0);
7681
7682         return (rc);
7683 }
7684
7685 static int
7686 in_range(int val, int lo, int hi)
7687 {
7688
7689         return (val < 0 || (val <= hi && val >= lo));
7690 }
7691
7692 static int
7693 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7694 {
7695         int fw_subcmd, fw_type, rc;
7696
7697         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7698         if (rc)
7699                 return (rc);
7700
7701         if (!(sc->flags & FULL_INIT_DONE)) {
7702                 rc = EAGAIN;
7703                 goto done;
7704         }
7705
7706         /*
7707          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7708          * sub-command and type are in common locations.)
7709          */
7710         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7711                 fw_subcmd = FW_SCHED_SC_CONFIG;
7712         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7713                 fw_subcmd = FW_SCHED_SC_PARAMS;
7714         else {
7715                 rc = EINVAL;
7716                 goto done;
7717         }
7718         if (p->type == SCHED_CLASS_TYPE_PACKET)
7719                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7720         else {
7721                 rc = EINVAL;
7722                 goto done;
7723         }
7724
7725         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7726                 /* Vet our parameters ..*/
7727                 if (p->u.config.minmax < 0) {
7728                         rc = EINVAL;
7729                         goto done;
7730                 }
7731
7732                 /* And pass the request to the firmware ...*/
7733                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7734                 goto done;
7735         }
7736
7737         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7738                 int fw_level;
7739                 int fw_mode;
7740                 int fw_rateunit;
7741                 int fw_ratemode;
7742
7743                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7744                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7745                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7746                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7747                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7748                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7749                 else {
7750                         rc = EINVAL;
7751                         goto done;
7752                 }
7753
7754                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7755                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7756                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7757                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7758                 else {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762
7763                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7764                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7765                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7766                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7767                 else {
7768                         rc = EINVAL;
7769                         goto done;
7770                 }
7771
7772                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7773                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7774                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7775                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7776                 else {
7777                         rc = EINVAL;
7778                         goto done;
7779                 }
7780
7781                 /* Vet our parameters ... */
7782                 if (!in_range(p->u.params.channel, 0, 3) ||
7783                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7784                     !in_range(p->u.params.minrate, 0, 10000000) ||
7785                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7786                     !in_range(p->u.params.weight, 0, 100)) {
7787                         rc = ERANGE;
7788                         goto done;
7789                 }
7790
7791                 /*
7792                  * Translate any unset parameters into the firmware's
7793                  * nomenclature and/or fail the call if the parameters
7794                  * are required ...
7795                  */
7796                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7797                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7798                         rc = EINVAL;
7799                         goto done;
7800                 }
7801                 if (p->u.params.minrate < 0)
7802                         p->u.params.minrate = 0;
7803                 if (p->u.params.maxrate < 0) {
7804                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7805                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7806                                 rc = EINVAL;
7807                                 goto done;
7808                         } else
7809                                 p->u.params.maxrate = 0;
7810                 }
7811                 if (p->u.params.weight < 0) {
7812                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7813                                 rc = EINVAL;
7814                                 goto done;
7815                         } else
7816                                 p->u.params.weight = 0;
7817                 }
7818                 if (p->u.params.pktsize < 0) {
7819                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7820                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7821                                 rc = EINVAL;
7822                                 goto done;
7823                         } else
7824                                 p->u.params.pktsize = 0;
7825                 }
7826
7827                 /* See what the firmware thinks of the request ... */
7828                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7829                     fw_rateunit, fw_ratemode, p->u.params.channel,
7830                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7831                     p->u.params.weight, p->u.params.pktsize, 1);
7832                 goto done;
7833         }
7834
7835         rc = EINVAL;
7836 done:
7837         end_synchronized_op(sc, 0);
7838         return (rc);
7839 }
7840
7841 static int
7842 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7843 {
7844         struct port_info *pi = NULL;
7845         struct sge_txq *txq;
7846         uint32_t fw_mnem, fw_queue, fw_class;
7847         int i, rc;
7848
7849         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7850         if (rc)
7851                 return (rc);
7852
7853         if (!(sc->flags & FULL_INIT_DONE)) {
7854                 rc = EAGAIN;
7855                 goto done;
7856         }
7857
7858         if (p->port >= sc->params.nports) {
7859                 rc = EINVAL;
7860                 goto done;
7861         }
7862
7863         pi = sc->port[p->port];
7864         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7865                 rc = EINVAL;
7866                 goto done;
7867         }
7868
7869         /*
7870          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7871          * Scheduling Class in this case).
7872          */
7873         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7874             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7875         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7876
7877         /*
7878          * If op.queue is non-negative, then we're only changing the scheduling
7879          * on a single specified TX queue.
7880          */
7881         if (p->queue >= 0) {
7882                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7883                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7884                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7885                     &fw_class);
7886                 goto done;
7887         }
7888
7889         /*
7890          * Change the scheduling on all the TX queues for the
7891          * interface.
7892          */
7893         for_each_txq(pi, i, txq) {
7894                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7895                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7896                     &fw_class);
7897                 if (rc)
7898                         goto done;
7899         }
7900
7901         rc = 0;
7902 done:
7903         end_synchronized_op(sc, 0);
7904         return (rc);
7905 }
7906
7907 int
7908 t4_os_find_pci_capability(struct adapter *sc, int cap)
7909 {
7910         int i;
7911
7912         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7913 }
7914
7915 int
7916 t4_os_pci_save_state(struct adapter *sc)
7917 {
7918         device_t dev;
7919         struct pci_devinfo *dinfo;
7920
7921         dev = sc->dev;
7922         dinfo = device_get_ivars(dev);
7923
7924         pci_cfg_save(dev, dinfo, 0);
7925         return (0);
7926 }
7927
7928 int
7929 t4_os_pci_restore_state(struct adapter *sc)
7930 {
7931         device_t dev;
7932         struct pci_devinfo *dinfo;
7933
7934         dev = sc->dev;
7935         dinfo = device_get_ivars(dev);
7936
7937         pci_cfg_restore(dev, dinfo);
7938         return (0);
7939 }
7940
7941 void
7942 t4_os_portmod_changed(const struct adapter *sc, int idx)
7943 {
7944         struct port_info *pi = sc->port[idx];
7945         static const char *mod_str[] = {
7946                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7947         };
7948
7949         build_medialist(pi, &pi->media);
7950 #ifdef DEV_NETMAP
7951         build_medialist(pi, &pi->nm_media);
7952 #endif
7953
7954         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7955                 if_printf(pi->ifp, "transceiver unplugged.\n");
7956         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7957                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7958         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7959                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7960         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7961                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7962                     mod_str[pi->mod_type]);
7963         } else {
7964                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7965                     pi->mod_type);
7966         }
7967 }
7968
7969 void
7970 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7971 {
7972         struct port_info *pi = sc->port[idx];
7973         struct ifnet *ifp = pi->ifp;
7974
7975         if (link_stat) {
7976                 pi->linkdnrc = -1;
7977                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7978                 if_link_state_change(ifp, LINK_STATE_UP);
7979         } else {
7980                 if (reason >= 0)
7981                         pi->linkdnrc = reason;
7982                 if_link_state_change(ifp, LINK_STATE_DOWN);
7983         }
7984 }
7985
7986 void
7987 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7988 {
7989         struct adapter *sc;
7990
7991         sx_slock(&t4_list_lock);
7992         SLIST_FOREACH(sc, &t4_list, link) {
7993                 /*
7994                  * func should not make any assumptions about what state sc is
7995                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7996                  */
7997                 func(sc, arg);
7998         }
7999         sx_sunlock(&t4_list_lock);
8000 }
8001
8002 static int
8003 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8004 {
8005        return (0);
8006 }
8007
8008 static int
8009 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8010 {
8011        return (0);
8012 }
8013
8014 static int
8015 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8016     struct thread *td)
8017 {
8018         int rc;
8019         struct adapter *sc = dev->si_drv1;
8020
8021         rc = priv_check(td, PRIV_DRIVER);
8022         if (rc != 0)
8023                 return (rc);
8024
8025         switch (cmd) {
8026         case CHELSIO_T4_GETREG: {
8027                 struct t4_reg *edata = (struct t4_reg *)data;
8028
8029                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8030                         return (EFAULT);
8031
8032                 if (edata->size == 4)
8033                         edata->val = t4_read_reg(sc, edata->addr);
8034                 else if (edata->size == 8)
8035                         edata->val = t4_read_reg64(sc, edata->addr);
8036                 else
8037                         return (EINVAL);
8038
8039                 break;
8040         }
8041         case CHELSIO_T4_SETREG: {
8042                 struct t4_reg *edata = (struct t4_reg *)data;
8043
8044                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8045                         return (EFAULT);
8046
8047                 if (edata->size == 4) {
8048                         if (edata->val & 0xffffffff00000000)
8049                                 return (EINVAL);
8050                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8051                 } else if (edata->size == 8)
8052                         t4_write_reg64(sc, edata->addr, edata->val);
8053                 else
8054                         return (EINVAL);
8055                 break;
8056         }
8057         case CHELSIO_T4_REGDUMP: {
8058                 struct t4_regdump *regs = (struct t4_regdump *)data;
8059                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8060                 uint8_t *buf;
8061
8062                 if (regs->len < reglen) {
8063                         regs->len = reglen; /* hint to the caller */
8064                         return (ENOBUFS);
8065                 }
8066
8067                 regs->len = reglen;
8068                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8069                 t4_get_regs(sc, regs, buf);
8070                 rc = copyout(buf, regs->data, reglen);
8071                 free(buf, M_CXGBE);
8072                 break;
8073         }
8074         case CHELSIO_T4_GET_FILTER_MODE:
8075                 rc = get_filter_mode(sc, (uint32_t *)data);
8076                 break;
8077         case CHELSIO_T4_SET_FILTER_MODE:
8078                 rc = set_filter_mode(sc, *(uint32_t *)data);
8079                 break;
8080         case CHELSIO_T4_GET_FILTER:
8081                 rc = get_filter(sc, (struct t4_filter *)data);
8082                 break;
8083         case CHELSIO_T4_SET_FILTER:
8084                 rc = set_filter(sc, (struct t4_filter *)data);
8085                 break;
8086         case CHELSIO_T4_DEL_FILTER:
8087                 rc = del_filter(sc, (struct t4_filter *)data);
8088                 break;
8089         case CHELSIO_T4_GET_SGE_CONTEXT:
8090                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8091                 break;
8092         case CHELSIO_T4_LOAD_FW:
8093                 rc = load_fw(sc, (struct t4_data *)data);
8094                 break;
8095         case CHELSIO_T4_GET_MEM:
8096                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8097                 break;
8098         case CHELSIO_T4_GET_I2C:
8099                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8100                 break;
8101         case CHELSIO_T4_CLEAR_STATS: {
8102                 int i;
8103                 u_int port_id = *(uint32_t *)data;
8104                 struct port_info *pi;
8105
8106                 if (port_id >= sc->params.nports)
8107                         return (EINVAL);
8108                 pi = sc->port[port_id];
8109
8110                 /* MAC stats */
8111                 t4_clr_port_stats(sc, pi->tx_chan);
8112                 pi->tx_parse_error = 0;
8113
8114                 if (pi->flags & PORT_INIT_DONE) {
8115                         struct sge_rxq *rxq;
8116                         struct sge_txq *txq;
8117                         struct sge_wrq *wrq;
8118
8119                         for_each_rxq(pi, i, rxq) {
8120 #if defined(INET) || defined(INET6)
8121                                 rxq->lro.lro_queued = 0;
8122                                 rxq->lro.lro_flushed = 0;
8123 #endif
8124                                 rxq->rxcsum = 0;
8125                                 rxq->vlan_extraction = 0;
8126                         }
8127
8128                         for_each_txq(pi, i, txq) {
8129                                 txq->txcsum = 0;
8130                                 txq->tso_wrs = 0;
8131                                 txq->vlan_insertion = 0;
8132                                 txq->imm_wrs = 0;
8133                                 txq->sgl_wrs = 0;
8134                                 txq->txpkt_wrs = 0;
8135                                 txq->txpkts0_wrs = 0;
8136                                 txq->txpkts1_wrs = 0;
8137                                 txq->txpkts0_pkts = 0;
8138                                 txq->txpkts1_pkts = 0;
8139                                 mp_ring_reset_stats(txq->r);
8140                         }
8141
8142 #ifdef TCP_OFFLOAD
8143                         /* nothing to clear for each ofld_rxq */
8144
8145                         for_each_ofld_txq(pi, i, wrq) {
8146                                 wrq->tx_wrs_direct = 0;
8147                                 wrq->tx_wrs_copied = 0;
8148                         }
8149 #endif
8150                         wrq = &sc->sge.ctrlq[pi->port_id];
8151                         wrq->tx_wrs_direct = 0;
8152                         wrq->tx_wrs_copied = 0;
8153                 }
8154                 break;
8155         }
8156         case CHELSIO_T4_SCHED_CLASS:
8157                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8158                 break;
8159         case CHELSIO_T4_SCHED_QUEUE:
8160                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8161                 break;
8162         case CHELSIO_T4_GET_TRACER:
8163                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8164                 break;
8165         case CHELSIO_T4_SET_TRACER:
8166                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8167                 break;
8168         default:
8169                 rc = EINVAL;
8170         }
8171
8172         return (rc);
8173 }
8174
8175 #ifdef TCP_OFFLOAD
8176 void
8177 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8178     const unsigned int *pgsz_order)
8179 {
8180         struct port_info *pi = ifp->if_softc;
8181         struct adapter *sc = pi->adapter;
8182
8183         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8184         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8185                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8186                 V_HPZ3(pgsz_order[3]));
8187 }
8188
8189 static int
8190 toe_capability(struct port_info *pi, int enable)
8191 {
8192         int rc;
8193         struct adapter *sc = pi->adapter;
8194
8195         ASSERT_SYNCHRONIZED_OP(sc);
8196
8197         if (!is_offload(sc))
8198                 return (ENODEV);
8199
8200         if (enable) {
8201                 if (!(sc->flags & FULL_INIT_DONE)) {
8202                         rc = cxgbe_init_synchronized(pi);
8203                         if (rc)
8204                                 return (rc);
8205                 }
8206
8207                 if (isset(&sc->offload_map, pi->port_id))
8208                         return (0);
8209
8210                 if (!(sc->flags & TOM_INIT_DONE)) {
8211                         rc = t4_activate_uld(sc, ULD_TOM);
8212                         if (rc == EAGAIN) {
8213                                 log(LOG_WARNING,
8214                                     "You must kldload t4_tom.ko before trying "
8215                                     "to enable TOE on a cxgbe interface.\n");
8216                         }
8217                         if (rc != 0)
8218                                 return (rc);
8219                         KASSERT(sc->tom_softc != NULL,
8220                             ("%s: TOM activated but softc NULL", __func__));
8221                         KASSERT(sc->flags & TOM_INIT_DONE,
8222                             ("%s: TOM activated but flag not set", __func__));
8223                 }
8224
8225                 setbit(&sc->offload_map, pi->port_id);
8226         } else {
8227                 if (!isset(&sc->offload_map, pi->port_id))
8228                         return (0);
8229
8230                 KASSERT(sc->flags & TOM_INIT_DONE,
8231                     ("%s: TOM never initialized?", __func__));
8232                 clrbit(&sc->offload_map, pi->port_id);
8233         }
8234
8235         return (0);
8236 }
8237
8238 /*
8239  * Add an upper layer driver to the global list.
8240  */
8241 int
8242 t4_register_uld(struct uld_info *ui)
8243 {
8244         int rc = 0;
8245         struct uld_info *u;
8246
8247         sx_xlock(&t4_uld_list_lock);
8248         SLIST_FOREACH(u, &t4_uld_list, link) {
8249             if (u->uld_id == ui->uld_id) {
8250                     rc = EEXIST;
8251                     goto done;
8252             }
8253         }
8254
8255         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8256         ui->refcount = 0;
8257 done:
8258         sx_xunlock(&t4_uld_list_lock);
8259         return (rc);
8260 }
8261
8262 int
8263 t4_unregister_uld(struct uld_info *ui)
8264 {
8265         int rc = EINVAL;
8266         struct uld_info *u;
8267
8268         sx_xlock(&t4_uld_list_lock);
8269
8270         SLIST_FOREACH(u, &t4_uld_list, link) {
8271             if (u == ui) {
8272                     if (ui->refcount > 0) {
8273                             rc = EBUSY;
8274                             goto done;
8275                     }
8276
8277                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8278                     rc = 0;
8279                     goto done;
8280             }
8281         }
8282 done:
8283         sx_xunlock(&t4_uld_list_lock);
8284         return (rc);
8285 }
8286
8287 int
8288 t4_activate_uld(struct adapter *sc, int id)
8289 {
8290         int rc = EAGAIN;
8291         struct uld_info *ui;
8292
8293         ASSERT_SYNCHRONIZED_OP(sc);
8294
8295         sx_slock(&t4_uld_list_lock);
8296
8297         SLIST_FOREACH(ui, &t4_uld_list, link) {
8298                 if (ui->uld_id == id) {
8299                         if (!(sc->flags & FULL_INIT_DONE)) {
8300                                 rc = adapter_full_init(sc);
8301                                 if (rc != 0)
8302                                         goto done;
8303                         }
8304
8305                         rc = ui->activate(sc);
8306                         if (rc == 0)
8307                                 ui->refcount++;
8308                         goto done;
8309                 }
8310         }
8311 done:
8312         sx_sunlock(&t4_uld_list_lock);
8313
8314         return (rc);
8315 }
8316
8317 int
8318 t4_deactivate_uld(struct adapter *sc, int id)
8319 {
8320         int rc = EINVAL;
8321         struct uld_info *ui;
8322
8323         ASSERT_SYNCHRONIZED_OP(sc);
8324
8325         sx_slock(&t4_uld_list_lock);
8326
8327         SLIST_FOREACH(ui, &t4_uld_list, link) {
8328                 if (ui->uld_id == id) {
8329                         rc = ui->deactivate(sc);
8330                         if (rc == 0)
8331                                 ui->refcount--;
8332                         goto done;
8333                 }
8334         }
8335 done:
8336         sx_sunlock(&t4_uld_list_lock);
8337
8338         return (rc);
8339 }
8340 #endif
8341
8342 /*
8343  * Come up with reasonable defaults for some of the tunables, provided they're
8344  * not set by the user (in which case we'll use the values as is).
8345  */
8346 static void
8347 tweak_tunables(void)
8348 {
8349         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8350
8351         if (t4_ntxq10g < 1)
8352                 t4_ntxq10g = min(nc, NTXQ_10G);
8353
8354         if (t4_ntxq1g < 1)
8355                 t4_ntxq1g = min(nc, NTXQ_1G);
8356
8357         if (t4_nrxq10g < 1)
8358                 t4_nrxq10g = min(nc, NRXQ_10G);
8359
8360         if (t4_nrxq1g < 1)
8361                 t4_nrxq1g = min(nc, NRXQ_1G);
8362
8363 #ifdef TCP_OFFLOAD
8364         if (t4_nofldtxq10g < 1)
8365                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8366
8367         if (t4_nofldtxq1g < 1)
8368                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8369
8370         if (t4_nofldrxq10g < 1)
8371                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8372
8373         if (t4_nofldrxq1g < 1)
8374                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8375
8376         if (t4_toecaps_allowed == -1)
8377                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8378 #else
8379         if (t4_toecaps_allowed == -1)
8380                 t4_toecaps_allowed = 0;
8381 #endif
8382
8383 #ifdef DEV_NETMAP
8384         if (t4_nnmtxq10g < 1)
8385                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8386
8387         if (t4_nnmtxq1g < 1)
8388                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8389
8390         if (t4_nnmrxq10g < 1)
8391                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8392
8393         if (t4_nnmrxq1g < 1)
8394                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8395 #endif
8396
8397         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8398                 t4_tmr_idx_10g = TMR_IDX_10G;
8399
8400         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8401                 t4_pktc_idx_10g = PKTC_IDX_10G;
8402
8403         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8404                 t4_tmr_idx_1g = TMR_IDX_1G;
8405
8406         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8407                 t4_pktc_idx_1g = PKTC_IDX_1G;
8408
8409         if (t4_qsize_txq < 128)
8410                 t4_qsize_txq = 128;
8411
8412         if (t4_qsize_rxq < 128)
8413                 t4_qsize_rxq = 128;
8414         while (t4_qsize_rxq & 7)
8415                 t4_qsize_rxq++;
8416
8417         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8418 }
8419
8420 static struct sx mlu;   /* mod load unload */
8421 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8422
8423 static int
8424 mod_event(module_t mod, int cmd, void *arg)
8425 {
8426         int rc = 0;
8427         static int loaded = 0;
8428
8429         switch (cmd) {
8430         case MOD_LOAD:
8431                 sx_xlock(&mlu);
8432                 if (loaded++ == 0) {
8433                         t4_sge_modload();
8434                         sx_init(&t4_list_lock, "T4/T5 adapters");
8435                         SLIST_INIT(&t4_list);
8436 #ifdef TCP_OFFLOAD
8437                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8438                         SLIST_INIT(&t4_uld_list);
8439 #endif
8440                         t4_tracer_modload();
8441                         tweak_tunables();
8442                 }
8443                 sx_xunlock(&mlu);
8444                 break;
8445
8446         case MOD_UNLOAD:
8447                 sx_xlock(&mlu);
8448                 if (--loaded == 0) {
8449                         int tries;
8450
8451                         sx_slock(&t4_list_lock);
8452                         if (!SLIST_EMPTY(&t4_list)) {
8453                                 rc = EBUSY;
8454                                 sx_sunlock(&t4_list_lock);
8455                                 goto done_unload;
8456                         }
8457 #ifdef TCP_OFFLOAD
8458                         sx_slock(&t4_uld_list_lock);
8459                         if (!SLIST_EMPTY(&t4_uld_list)) {
8460                                 rc = EBUSY;
8461                                 sx_sunlock(&t4_uld_list_lock);
8462                                 sx_sunlock(&t4_list_lock);
8463                                 goto done_unload;
8464                         }
8465 #endif
8466                         tries = 0;
8467                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8468                                 uprintf("%ju clusters with custom free routine "
8469                                     "still is use.\n", t4_sge_extfree_refs());
8470                                 pause("t4unload", 2 * hz);
8471                         }
8472 #ifdef TCP_OFFLOAD
8473                         sx_sunlock(&t4_uld_list_lock);
8474 #endif
8475                         sx_sunlock(&t4_list_lock);
8476
8477                         if (t4_sge_extfree_refs() == 0) {
8478                                 t4_tracer_modunload();
8479 #ifdef TCP_OFFLOAD
8480                                 sx_destroy(&t4_uld_list_lock);
8481 #endif
8482                                 sx_destroy(&t4_list_lock);
8483                                 t4_sge_modunload();
8484                                 loaded = 0;
8485                         } else {
8486                                 rc = EBUSY;
8487                                 loaded++;       /* undo earlier decrement */
8488                         }
8489                 }
8490 done_unload:
8491                 sx_xunlock(&mlu);
8492                 break;
8493         }
8494
8495         return (rc);
8496 }
8497
8498 static devclass_t t4_devclass, t5_devclass;
8499 static devclass_t cxgbe_devclass, cxl_devclass;
8500
8501 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8502 MODULE_VERSION(t4nex, 1);
8503 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8504
8505 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8506 MODULE_VERSION(t5nex, 1);
8507 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8508
8509 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8510 MODULE_VERSION(cxgbe, 1);
8511
8512 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8513 MODULE_VERSION(cxl, 1);