sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69 #include "t4_mp_ring.h"
  70
  71 /* T4 bus driver interface */
  72 static int t4_probe(device_t);
  73 static int t4_attach(device_t);
  74 static int t4_detach(device_t);
  75 static device_method_t t4_methods[] = {
  76         DEVMETHOD(device_probe,         t4_probe),
  77         DEVMETHOD(device_attach,        t4_attach),
  78         DEVMETHOD(device_detach,        t4_detach),
  79
  80         DEVMETHOD_END
  81 };
  82 static driver_t t4_driver = {
  83         "t4nex",
  84         t4_methods,
  85         sizeof(struct adapter)
  86 };
  87
  88
  89 /* T4 port (cxgbe) interface */
  90 static int cxgbe_probe(device_t);
  91 static int cxgbe_attach(device_t);
  92 static int cxgbe_detach(device_t);
  93 static device_method_t cxgbe_methods[] = {
  94         DEVMETHOD(device_probe,         cxgbe_probe),
  95         DEVMETHOD(device_attach,        cxgbe_attach),
  96         DEVMETHOD(device_detach,        cxgbe_detach),
  97         { 0, 0 }
  98 };
  99 static driver_t cxgbe_driver = {
 100         "cxgbe",
 101         cxgbe_methods,
 102         sizeof(struct port_info)
 103 };
 104
 105 static d_ioctl_t t4_ioctl;
 106 static d_open_t t4_open;
 107 static d_close_t t4_close;
 108
 109 static struct cdevsw t4_cdevsw = {
 110        .d_version = D_VERSION,
 111        .d_flags = 0,
 112        .d_open = t4_open,
 113        .d_close = t4_close,
 114        .d_ioctl = t4_ioctl,
 115        .d_name = "t4nex",
 116 };
 117
 118 /* T5 bus driver interface */
 119 static int t5_probe(device_t);
 120 static device_method_t t5_methods[] = {
 121         DEVMETHOD(device_probe,         t5_probe),
 122         DEVMETHOD(device_attach,        t4_attach),
 123         DEVMETHOD(device_detach,        t4_detach),
 124
 125         DEVMETHOD_END
 126 };
 127 static driver_t t5_driver = {
 128         "t5nex",
 129         t5_methods,
 130         sizeof(struct adapter)
 131 };
 132
 133
 134 /* T5 port (cxl) interface */
 135 static driver_t cxl_driver = {
 136         "cxl",
 137         cxgbe_methods,
 138         sizeof(struct port_info)
 139 };
 140
 141 static struct cdevsw t5_cdevsw = {
 142        .d_version = D_VERSION,
 143        .d_flags = 0,
 144        .d_open = t4_open,
 145        .d_close = t4_close,
 146        .d_ioctl = t4_ioctl,
 147        .d_name = "t5nex",
 148 };
 149
 150 /* ifnet + media interface */
 151 static void cxgbe_init(void *);
 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 154 static void cxgbe_qflush(struct ifnet *);
 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 156 static int cxgbe_media_change(struct ifnet *);
 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 158
 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 160
 161 /*
 162  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 163  * then ADAPTER_LOCK, then t4_uld_list_lock.
 164  */
 165 static struct sx t4_list_lock;
 166 SLIST_HEAD(, adapter) t4_list;
 167 #ifdef TCP_OFFLOAD
 168 static struct sx t4_uld_list_lock;
 169 SLIST_HEAD(, uld_info) t4_uld_list;
 170 #endif
 171
 172 /*
 173  * Tunables.  See tweak_tunables() too.
 174  *
 175  * Each tunable is set to a default value here if it's known at compile-time.
 176  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 177  * provide a reasonable default when the driver is loaded.
 178  *
 179  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 180  * T5 are under hw.cxl.
 181  */
 182
 183 /*
 184  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 185  */
 186 #define NTXQ_10G 16
 187 static int t4_ntxq10g = -1;
 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 189
 190 #define NRXQ_10G 8
 191 static int t4_nrxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 193
 194 #define NTXQ_1G 4
 195 static int t4_ntxq1g = -1;
 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 197
 198 #define NRXQ_1G 2
 199 static int t4_nrxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 201
 202 static int t4_rsrv_noflowq = 0;
 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 204
 205 #ifdef TCP_OFFLOAD
 206 #define NOFLDTXQ_10G 8
 207 static int t4_nofldtxq10g = -1;
 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 209
 210 #define NOFLDRXQ_10G 2
 211 static int t4_nofldrxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 213
 214 #define NOFLDTXQ_1G 2
 215 static int t4_nofldtxq1g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 217
 218 #define NOFLDRXQ_1G 1
 219 static int t4_nofldrxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 221 #endif
 222
 223 #ifdef DEV_NETMAP
 224 #define NNMTXQ_10G 2
 225 static int t4_nnmtxq10g = -1;
 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 227
 228 #define NNMRXQ_10G 2
 229 static int t4_nnmrxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 231
 232 #define NNMTXQ_1G 1
 233 static int t4_nnmtxq1g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 235
 236 #define NNMRXQ_1G 1
 237 static int t4_nnmrxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 239 #endif
 240
 241 /*
 242  * Holdoff parameters for 10G and 1G ports.
 243  */
 244 #define TMR_IDX_10G 1
 245 static int t4_tmr_idx_10g = TMR_IDX_10G;
 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 247
 248 #define PKTC_IDX_10G (-1)
 249 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 251
 252 #define TMR_IDX_1G 1
 253 static int t4_tmr_idx_1g = TMR_IDX_1G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 255
 256 #define PKTC_IDX_1G (-1)
 257 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 259
 260 /*
 261  * Size (# of entries) of each tx and rx queue.
 262  */
 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 265
 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 268
 269 /*
 270  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 271  */
 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 274
 275 /*
 276  * Configuration file.
 277  */
 278 #define DEFAULT_CF      "default"
 279 #define FLASH_CF        "flash"
 280 #define UWIRE_CF        "uwire"
 281 #define FPGA_CF         "fpga"
 282 static char t4_cfg_file[32] = DEFAULT_CF;
 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 284
 285 /*
 286  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 287  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 288  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 289  *            mark or when signalled to do so, 0 to never emit PAUSE.
 290  */
 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 293
 294 /*
 295  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 296  * encouraged respectively).
 297  */
 298 static unsigned int t4_fw_install = 1;
 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 300
 301 /*
 302  * ASIC features that will be used.  Disable the ones you don't want so that the
 303  * chip resources aren't wasted on features that will not be used.
 304  */
 305 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 307
 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 310
 311 static int t4_toecaps_allowed = -1;
 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 313
 314 static int t4_rdmacaps_allowed = 0;
 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 316
 317 static int t4_iscsicaps_allowed = 0;
 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 319
 320 static int t4_fcoecaps_allowed = 0;
 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 322
 323 static int t5_write_combine = 0;
 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 325
 326 struct intrs_and_queues {
 327         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 328         uint16_t nirq;          /* Total # of vectors */
 329         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 330         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 331         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 332         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 333         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 334         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 335         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 336 #ifdef TCP_OFFLOAD
 337         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 338         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 339         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 340         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 341 #endif
 342 #ifdef DEV_NETMAP
 343         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 344         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 345         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 346         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 347 #endif
 348 };
 349
 350 struct filter_entry {
 351         uint32_t valid:1;       /* filter allocated and valid */
 352         uint32_t locked:1;      /* filter is administratively locked */
 353         uint32_t pending:1;     /* filter action is pending firmware reply */
 354         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 355         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 356
 357         struct t4_filter_specification fs;
 358 };
 359
 360 static int map_bars_0_and_4(struct adapter *);
 361 static int map_bar_2(struct adapter *);
 362 static void setup_memwin(struct adapter *);
 363 static int validate_mem_range(struct adapter *, uint32_t, int);
 364 static int fwmtype_to_hwmtype(int);
 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 366     uint32_t *);
 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 368 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 369 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 370     struct intrs_and_queues *);
 371 static int prep_firmware(struct adapter *);
 372 static int partition_resources(struct adapter *, const struct firmware *,
 373     const char *);
 374 static int get_params__pre_init(struct adapter *);
 375 static int get_params__post_init(struct adapter *);
 376 static int set_params__post_init(struct adapter *);
 377 static void t4_set_desc(struct adapter *);
 378 static void build_medialist(struct port_info *, struct ifmedia *);
 379 static int cxgbe_init_synchronized(struct port_info *);
 380 static int cxgbe_uninit_synchronized(struct port_info *);
 381 static int setup_intr_handlers(struct adapter *);
 382 static void quiesce_txq(struct adapter *, struct sge_txq *);
 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 384 static void quiesce_iq(struct adapter *, struct sge_iq *);
 385 static void quiesce_fl(struct adapter *, struct sge_fl *);
 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 387     driver_intr_t *, void *, char *);
 388 static int t4_free_irq(struct adapter *, struct irq *);
 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 390     unsigned int);
 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 393 static void cxgbe_tick(void *);
 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 396     struct mbuf *);
 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 398 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 399 static int t4_sysctls(struct adapter *);
 400 static int cxgbe_sysctls(struct port_info *);
 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 412 #ifdef SBUF_DRAIN
 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 438 #endif
 439 static uint32_t fconf_to_mode(uint32_t);
 440 static uint32_t mode_to_fconf(uint32_t);
 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 442 static int get_filter_mode(struct adapter *, uint32_t *);
 443 static int set_filter_mode(struct adapter *, uint32_t);
 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 445 static int get_filter(struct adapter *, struct t4_filter *);
 446 static int set_filter(struct adapter *, struct t4_filter *);
 447 static int del_filter(struct adapter *, struct t4_filter *);
 448 static void clear_filter(struct filter_entry *);
 449 static int set_filter_wr(struct adapter *, int);
 450 static int del_filter_wr(struct adapter *, int);
 451 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 452 static int load_fw(struct adapter *, struct t4_data *);
 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 454 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 455 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 457 #ifdef TCP_OFFLOAD
 458 static int toe_capability(struct port_info *, int);
 459 #endif
 460 static int mod_event(module_t, int, void *);
 461
 462 struct {
 463         uint16_t device;
 464         char *desc;
 465 } t4_pciids[] = {
 466         {0xa000, "Chelsio Terminator 4 FPGA"},
 467         {0x4400, "Chelsio T440-dbg"},
 468         {0x4401, "Chelsio T420-CR"},
 469         {0x4402, "Chelsio T422-CR"},
 470         {0x4403, "Chelsio T440-CR"},
 471         {0x4404, "Chelsio T420-BCH"},
 472         {0x4405, "Chelsio T440-BCH"},
 473         {0x4406, "Chelsio T440-CH"},
 474         {0x4407, "Chelsio T420-SO"},
 475         {0x4408, "Chelsio T420-CX"},
 476         {0x4409, "Chelsio T420-BT"},
 477         {0x440a, "Chelsio T404-BT"},
 478         {0x440e, "Chelsio T440-LP-CR"},
 479 }, t5_pciids[] = {
 480         {0xb000, "Chelsio Terminator 5 FPGA"},
 481         {0x5400, "Chelsio T580-dbg"},
 482         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 483         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 484         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 485         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 486         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 487         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 488         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 489         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 490         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 491         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 492         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 493         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 494         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 495 #ifdef notyet
 496         {0x5404,  "Chelsio T520-BCH"},
 497         {0x5405,  "Chelsio T540-BCH"},
 498         {0x5406,  "Chelsio T540-CH"},
 499         {0x5408,  "Chelsio T520-CX"},
 500         {0x540b,  "Chelsio B520-SR"},
 501         {0x540c,  "Chelsio B504-BT"},
 502         {0x540f,  "Chelsio Amsterdam"},
 503         {0x5413,  "Chelsio T580-CHR"},
 504 #endif
 505 };
 506
 507 #ifdef TCP_OFFLOAD
 508 /*
 509  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 510  * exactly the same for both rxq and ofld_rxq.
 511  */
 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 514 #endif
 515
 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 519
 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 521
 522 static int
 523 t4_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xa000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t4_pciids); i++) {
 538                 if (d == t4_pciids[i].device) {
 539                         device_set_desc(dev, t4_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t5_probe(device_t dev)
 549 {
 550         int i;
 551         uint16_t v = pci_get_vendor(dev);
 552         uint16_t d = pci_get_device(dev);
 553         uint8_t f = pci_get_function(dev);
 554
 555         if (v != PCI_VENDOR_ID_CHELSIO)
 556                 return (ENXIO);
 557
 558         /* Attach only to PF0 of the FPGA */
 559         if (d == 0xb000 && f != 0)
 560                 return (ENXIO);
 561
 562         for (i = 0; i < nitems(t5_pciids); i++) {
 563                 if (d == t5_pciids[i].device) {
 564                         device_set_desc(dev, t5_pciids[i].desc);
 565                         return (BUS_PROBE_DEFAULT);
 566                 }
 567         }
 568
 569         return (ENXIO);
 570 }
 571
 572 static int
 573 t4_attach(device_t dev)
 574 {
 575         struct adapter *sc;
 576         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 577         struct intrs_and_queues iaq;
 578         struct sge *s;
 579 #ifdef TCP_OFFLOAD
 580         int ofld_rqidx, ofld_tqidx;
 581 #endif
 582 #ifdef DEV_NETMAP
 583         int nm_rqidx, nm_tqidx;
 584 #endif
 585         const char *pcie_ts;
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671 #if defined(__i386__)
 672         if ((cpu_feature & CPUID_CX8) == 0) {
 673                 device_printf(dev, "64 bit atomics not available.\n");
 674                 rc = ENOTSUP;
 675                 goto done;
 676         }
 677 #endif
 678
 679         /* Prepare the firmware for operation */
 680         rc = prep_firmware(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = get_params__post_init(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = set_params__post_init(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         rc = map_bar_2(sc);
 693         if (rc != 0)
 694                 goto done; /* error message displayed already */
 695
 696         rc = t4_create_dma_tag(sc);
 697         if (rc != 0)
 698                 goto done; /* error message displayed already */
 699
 700         /*
 701          * First pass over all the ports - allocate VIs and initialize some
 702          * basic parameters like mac address, port type, etc.  We also figure
 703          * out whether a port is 10G or 1G and use that information when
 704          * calculating how many interrupts to attempt to allocate.
 705          */
 706         n10g = n1g = 0;
 707         for_each_port(sc, i) {
 708                 struct port_info *pi;
 709
 710                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 711                 sc->port[i] = pi;
 712
 713                 /* These must be set before t4_port_init */
 714                 pi->adapter = sc;
 715                 pi->port_id = i;
 716
 717                 /* Allocate the vi and initialize parameters like mac addr */
 718                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 719                 if (rc != 0) {
 720                         device_printf(dev, "unable to initialize port %d: %d\n",
 721                             i, rc);
 722                         free(pi, M_CXGBE);
 723                         sc->port[i] = NULL;
 724                         goto done;
 725                 }
 726
 727                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 728                 pi->link_cfg.requested_fc |= t4_pause_settings;
 729                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 730                 pi->link_cfg.fc |= t4_pause_settings;
 731
 732                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 733                 if (rc != 0) {
 734                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 735                         free(pi, M_CXGBE);
 736                         sc->port[i] = NULL;
 737                         goto done;
 738                 }
 739
 740                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 741                     device_get_nameunit(dev), i);
 742                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 743                 sc->chan_map[pi->tx_chan] = i;
 744
 745                 if (is_10G_port(pi) || is_40G_port(pi)) {
 746                         n10g++;
 747                         pi->tmr_idx = t4_tmr_idx_10g;
 748                         pi->pktc_idx = t4_pktc_idx_10g;
 749                 } else {
 750                         n1g++;
 751                         pi->tmr_idx = t4_tmr_idx_1g;
 752                         pi->pktc_idx = t4_pktc_idx_1g;
 753                 }
 754
 755                 pi->xact_addr_filt = -1;
 756                 pi->linkdnrc = -1;
 757
 758                 pi->qsize_rxq = t4_qsize_rxq;
 759                 pi->qsize_txq = t4_qsize_txq;
 760
 761                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 762                 if (pi->dev == NULL) {
 763                         device_printf(dev,
 764                             "failed to add device for port %d.\n", i);
 765                         rc = ENXIO;
 766                         goto done;
 767                 }
 768                 device_set_softc(pi->dev, pi);
 769         }
 770
 771         /*
 772          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 773          */
 774         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 775         if (rc != 0)
 776                 goto done; /* error message displayed already */
 777
 778         sc->intr_type = iaq.intr_type;
 779         sc->intr_count = iaq.nirq;
 780
 781         s = &sc->sge;
 782         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 783         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 784         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 785         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 786         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 787 #ifdef TCP_OFFLOAD
 788         if (is_offload(sc)) {
 789                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 790                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 791                 s->neq += s->nofldtxq + s->nofldrxq;
 792                 s->niq += s->nofldrxq;
 793
 794                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 795                     M_CXGBE, M_ZERO | M_WAITOK);
 796                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 797                     M_CXGBE, M_ZERO | M_WAITOK);
 798         }
 799 #endif
 800 #ifdef DEV_NETMAP
 801         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 802         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 803         s->neq += s->nnmtxq + s->nnmrxq;
 804         s->niq += s->nnmrxq;
 805
 806         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 807             M_CXGBE, M_ZERO | M_WAITOK);
 808         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 809             M_CXGBE, M_ZERO | M_WAITOK);
 810 #endif
 811
 812         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 817             M_ZERO | M_WAITOK);
 818         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 819             M_ZERO | M_WAITOK);
 820         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 821             M_ZERO | M_WAITOK);
 822
 823         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 824             M_ZERO | M_WAITOK);
 825
 826         t4_init_l2t(sc, M_WAITOK);
 827
 828         /*
 829          * Second pass over the ports.  This time we know the number of rx and
 830          * tx queues that each port should get.
 831          */
 832         rqidx = tqidx = 0;
 833 #ifdef TCP_OFFLOAD
 834         ofld_rqidx = ofld_tqidx = 0;
 835 #endif
 836 #ifdef DEV_NETMAP
 837         nm_rqidx = nm_tqidx = 0;
 838 #endif
 839         for_each_port(sc, i) {
 840                 struct port_info *pi = sc->port[i];
 841
 842                 if (pi == NULL)
 843                         continue;
 844
 845                 pi->first_rxq = rqidx;
 846                 pi->first_txq = tqidx;
 847                 if (is_10G_port(pi) || is_40G_port(pi)) {
 848                         pi->flags |= iaq.intr_flags_10g;
 849                         pi->nrxq = iaq.nrxq10g;
 850                         pi->ntxq = iaq.ntxq10g;
 851                 } else {
 852                         pi->flags |= iaq.intr_flags_1g;
 853                         pi->nrxq = iaq.nrxq1g;
 854                         pi->ntxq = iaq.ntxq1g;
 855                 }
 856
 857                 if (pi->ntxq > 1)
 858                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 859                 else
 860                         pi->rsrv_noflowq = 0;
 861
 862                 rqidx += pi->nrxq;
 863                 tqidx += pi->ntxq;
 864 #ifdef TCP_OFFLOAD
 865                 if (is_offload(sc)) {
 866                         pi->first_ofld_rxq = ofld_rqidx;
 867                         pi->first_ofld_txq = ofld_tqidx;
 868                         if (is_10G_port(pi) || is_40G_port(pi)) {
 869                                 pi->nofldrxq = iaq.nofldrxq10g;
 870                                 pi->nofldtxq = iaq.nofldtxq10g;
 871                         } else {
 872                                 pi->nofldrxq = iaq.nofldrxq1g;
 873                                 pi->nofldtxq = iaq.nofldtxq1g;
 874                         }
 875                         ofld_rqidx += pi->nofldrxq;
 876                         ofld_tqidx += pi->nofldtxq;
 877                 }
 878 #endif
 879 #ifdef DEV_NETMAP
 880                 pi->first_nm_rxq = nm_rqidx;
 881                 pi->first_nm_txq = nm_tqidx;
 882                 if (is_10G_port(pi) || is_40G_port(pi)) {
 883                         pi->nnmrxq = iaq.nnmrxq10g;
 884                         pi->nnmtxq = iaq.nnmtxq10g;
 885                 } else {
 886                         pi->nnmrxq = iaq.nnmrxq1g;
 887                         pi->nnmtxq = iaq.nnmtxq1g;
 888                 }
 889                 nm_rqidx += pi->nnmrxq;
 890                 nm_tqidx += pi->nnmtxq;
 891 #endif
 892         }
 893
 894         rc = setup_intr_handlers(sc);
 895         if (rc != 0) {
 896                 device_printf(dev,
 897                     "failed to setup interrupt handlers: %d\n", rc);
 898                 goto done;
 899         }
 900
 901         rc = bus_generic_attach(dev);
 902         if (rc != 0) {
 903                 device_printf(dev,
 904                     "failed to attach all child ports: %d\n", rc);
 905                 goto done;
 906         }
 907
 908         switch (sc->params.pci.speed) {
 909                 case 0x1:
 910                         pcie_ts = "2.5";
 911                         break;
 912                 case 0x2:
 913                         pcie_ts = "5.0";
 914                         break;
 915                 case 0x3:
 916                         pcie_ts = "8.0";
 917                         break;
 918                 default:
 919                         pcie_ts = "??";
 920                         break;
 921         }
 922         device_printf(dev,
 923             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 924             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 925             sc->params.nports, sc->intr_count,
 926             sc->intr_type == INTR_MSIX ? "MSI-X" :
 927             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 928             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 929
 930         t4_set_desc(sc);
 931
 932 done:
 933         if (rc != 0 && sc->cdev) {
 934                 /* cdev was created and so cxgbetool works; recover that way. */
 935                 device_printf(dev,
 936                     "error during attach, adapter is now in recovery mode.\n");
 937                 rc = 0;
 938         }
 939
 940         if (rc != 0)
 941                 t4_detach(dev);
 942         else
 943                 t4_sysctls(sc);
 944
 945         return (rc);
 946 }
 947
 948 /*
 949  * Idempotent
 950  */
 951 static int
 952 t4_detach(device_t dev)
 953 {
 954         struct adapter *sc;
 955         struct port_info *pi;
 956         int i, rc;
 957
 958         sc = device_get_softc(dev);
 959
 960         if (sc->flags & FULL_INIT_DONE)
 961                 t4_intr_disable(sc);
 962
 963         if (sc->cdev) {
 964                 destroy_dev(sc->cdev);
 965                 sc->cdev = NULL;
 966         }
 967
 968         rc = bus_generic_detach(dev);
 969         if (rc) {
 970                 device_printf(dev,
 971                     "failed to detach child devices: %d\n", rc);
 972                 return (rc);
 973         }
 974
 975         for (i = 0; i < sc->intr_count; i++)
 976                 t4_free_irq(sc, &sc->irq[i]);
 977
 978         for (i = 0; i < MAX_NPORTS; i++) {
 979                 pi = sc->port[i];
 980                 if (pi) {
 981                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 982                         if (pi->dev)
 983                                 device_delete_child(dev, pi->dev);
 984
 985                         mtx_destroy(&pi->pi_lock);
 986                         free(pi, M_CXGBE);
 987                 }
 988         }
 989
 990         if (sc->flags & FULL_INIT_DONE)
 991                 adapter_full_uninit(sc);
 992
 993         if (sc->flags & FW_OK)
 994                 t4_fw_bye(sc, sc->mbox);
 995
 996         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 997                 pci_release_msi(dev);
 998
 999         if (sc->regs_res)
1000                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1001                     sc->regs_res);
1002
1003         if (sc->udbs_res)
1004                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1005                     sc->udbs_res);
1006
1007         if (sc->msix_res)
1008                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1009                     sc->msix_res);
1010
1011         if (sc->l2t)
1012                 t4_free_l2t(sc->l2t);
1013
1014 #ifdef TCP_OFFLOAD
1015         free(sc->sge.ofld_rxq, M_CXGBE);
1016         free(sc->sge.ofld_txq, M_CXGBE);
1017 #endif
1018 #ifdef DEV_NETMAP
1019         free(sc->sge.nm_rxq, M_CXGBE);
1020         free(sc->sge.nm_txq, M_CXGBE);
1021 #endif
1022         free(sc->irq, M_CXGBE);
1023         free(sc->sge.rxq, M_CXGBE);
1024         free(sc->sge.txq, M_CXGBE);
1025         free(sc->sge.ctrlq, M_CXGBE);
1026         free(sc->sge.iqmap, M_CXGBE);
1027         free(sc->sge.eqmap, M_CXGBE);
1028         free(sc->tids.ftid_tab, M_CXGBE);
1029         t4_destroy_dma_tag(sc);
1030         if (mtx_initialized(&sc->sc_lock)) {
1031                 sx_xlock(&t4_list_lock);
1032                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1033                 sx_xunlock(&t4_list_lock);
1034                 mtx_destroy(&sc->sc_lock);
1035         }
1036
1037         if (mtx_initialized(&sc->tids.ftid_lock))
1038                 mtx_destroy(&sc->tids.ftid_lock);
1039         if (mtx_initialized(&sc->sfl_lock))
1040                 mtx_destroy(&sc->sfl_lock);
1041         if (mtx_initialized(&sc->ifp_lock))
1042                 mtx_destroy(&sc->ifp_lock);
1043         if (mtx_initialized(&sc->regwin_lock))
1044                 mtx_destroy(&sc->regwin_lock);
1045
1046         bzero(sc, sizeof(*sc));
1047
1048         return (0);
1049 }
1050
1051 static int
1052 cxgbe_probe(device_t dev)
1053 {
1054         char buf[128];
1055         struct port_info *pi = device_get_softc(dev);
1056
1057         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1058         device_set_desc_copy(dev, buf);
1059
1060         return (BUS_PROBE_DEFAULT);
1061 }
1062
1063 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1064     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1065     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1066 #define T4_CAP_ENABLE (T4_CAP)
1067
1068 static int
1069 cxgbe_attach(device_t dev)
1070 {
1071         struct port_info *pi = device_get_softc(dev);
1072         struct ifnet *ifp;
1073         char *s;
1074         int n, o;
1075
1076         /* Allocate an ifnet and set it up */
1077         ifp = if_alloc(IFT_ETHER);
1078         if (ifp == NULL) {
1079                 device_printf(dev, "Cannot allocate ifnet\n");
1080                 return (ENOMEM);
1081         }
1082         pi->ifp = ifp;
1083         ifp->if_softc = pi;
1084
1085         callout_init(&pi->tick, CALLOUT_MPSAFE);
1086
1087         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1088         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1089
1090         ifp->if_init = cxgbe_init;
1091         ifp->if_ioctl = cxgbe_ioctl;
1092         ifp->if_transmit = cxgbe_transmit;
1093         ifp->if_qflush = cxgbe_qflush;
1094         ifp->if_get_counter = cxgbe_get_counter;
1095
1096         ifp->if_capabilities = T4_CAP;
1097 #ifdef TCP_OFFLOAD
1098         if (is_offload(pi->adapter))
1099                 ifp->if_capabilities |= IFCAP_TOE;
1100 #endif
1101         ifp->if_capenable = T4_CAP_ENABLE;
1102         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1103             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1104
1105         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1106         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1107         ifp->if_hw_tsomaxsegsize = 65536;
1108
1109         /* Initialize ifmedia for this port */
1110         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1111             cxgbe_media_status);
1112         build_medialist(pi, &pi->media);
1113
1114         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1115             EVENTHANDLER_PRI_ANY);
1116
1117         ether_ifattach(ifp, pi->hw_addr);
1118
1119         n = 128;
1120         s = malloc(n, M_CXGBE, M_WAITOK);
1121         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1122         MPASS(n > o);
1123 #ifdef TCP_OFFLOAD
1124         if (is_offload(pi->adapter)) {
1125                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1126                     pi->nofldtxq, pi->nofldrxq);
1127                 MPASS(n > o);
1128         }
1129 #endif
1130 #ifdef DEV_NETMAP
1131         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1132             pi->nnmrxq);
1133         MPASS(n > o);
1134 #endif
1135         device_printf(dev, "%s\n", s);
1136         free(s, M_CXGBE);
1137
1138 #ifdef DEV_NETMAP
1139         /* nm_media handled here to keep implementation private to this file */
1140         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1141             cxgbe_media_status);
1142         build_medialist(pi, &pi->nm_media);
1143         create_netmap_ifnet(pi);        /* logs errors it something fails */
1144 #endif
1145         cxgbe_sysctls(pi);
1146
1147         return (0);
1148 }
1149
1150 static int
1151 cxgbe_detach(device_t dev)
1152 {
1153         struct port_info *pi = device_get_softc(dev);
1154         struct adapter *sc = pi->adapter;
1155         struct ifnet *ifp = pi->ifp;
1156
1157         /* Tell if_ioctl and if_init that the port is going away */
1158         ADAPTER_LOCK(sc);
1159         SET_DOOMED(pi);
1160         wakeup(&sc->flags);
1161         while (IS_BUSY(sc))
1162                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1163         SET_BUSY(sc);
1164 #ifdef INVARIANTS
1165         sc->last_op = "t4detach";
1166         sc->last_op_thr = curthread;
1167 #endif
1168         ADAPTER_UNLOCK(sc);
1169
1170         if (pi->flags & HAS_TRACEQ) {
1171                 sc->traceq = -1;        /* cloner should not create ifnet */
1172                 t4_tracer_port_detach(sc);
1173         }
1174
1175         if (pi->vlan_c)
1176                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1177
1178         PORT_LOCK(pi);
1179         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1180         callout_stop(&pi->tick);
1181         PORT_UNLOCK(pi);
1182         callout_drain(&pi->tick);
1183
1184         /* Let detach proceed even if these fail. */
1185         cxgbe_uninit_synchronized(pi);
1186         port_full_uninit(pi);
1187
1188         ifmedia_removeall(&pi->media);
1189         ether_ifdetach(pi->ifp);
1190         if_free(pi->ifp);
1191
1192 #ifdef DEV_NETMAP
1193         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1194         destroy_netmap_ifnet(pi);
1195 #endif
1196
1197         ADAPTER_LOCK(sc);
1198         CLR_BUSY(sc);
1199         wakeup(&sc->flags);
1200         ADAPTER_UNLOCK(sc);
1201
1202         return (0);
1203 }
1204
1205 static void
1206 cxgbe_init(void *arg)
1207 {
1208         struct port_info *pi = arg;
1209         struct adapter *sc = pi->adapter;
1210
1211         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1212                 return;
1213         cxgbe_init_synchronized(pi);
1214         end_synchronized_op(sc, 0);
1215 }
1216
1217 static int
1218 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1219 {
1220         int rc = 0, mtu, flags, can_sleep;
1221         struct port_info *pi = ifp->if_softc;
1222         struct adapter *sc = pi->adapter;
1223         struct ifreq *ifr = (struct ifreq *)data;
1224         uint32_t mask;
1225
1226         switch (cmd) {
1227         case SIOCSIFMTU:
1228                 mtu = ifr->ifr_mtu;
1229                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1230                         return (EINVAL);
1231
1232                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1233                 if (rc)
1234                         return (rc);
1235                 ifp->if_mtu = mtu;
1236                 if (pi->flags & PORT_INIT_DONE) {
1237                         t4_update_fl_bufsize(ifp);
1238                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1239                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1240                 }
1241                 end_synchronized_op(sc, 0);
1242                 break;
1243
1244         case SIOCSIFFLAGS:
1245                 can_sleep = 0;
1246 redo_sifflags:
1247                 rc = begin_synchronized_op(sc, pi,
1248                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1249                 if (rc)
1250                         return (rc);
1251
1252                 if (ifp->if_flags & IFF_UP) {
1253                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1254                                 flags = pi->if_flags;
1255                                 if ((ifp->if_flags ^ flags) &
1256                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1257                                         if (can_sleep == 1) {
1258                                                 end_synchronized_op(sc, 0);
1259                                                 can_sleep = 0;
1260                                                 goto redo_sifflags;
1261                                         }
1262                                         rc = update_mac_settings(ifp,
1263                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1264                                 }
1265                         } else {
1266                                 if (can_sleep == 0) {
1267                                         end_synchronized_op(sc, LOCK_HELD);
1268                                         can_sleep = 1;
1269                                         goto redo_sifflags;
1270                                 }
1271                                 rc = cxgbe_init_synchronized(pi);
1272                         }
1273                         pi->if_flags = ifp->if_flags;
1274                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1275                         if (can_sleep == 0) {
1276                                 end_synchronized_op(sc, LOCK_HELD);
1277                                 can_sleep = 1;
1278                                 goto redo_sifflags;
1279                         }
1280                         rc = cxgbe_uninit_synchronized(pi);
1281                 }
1282                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1283                 break;
1284
1285         case SIOCADDMULTI:
1286         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1287                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1288                 if (rc)
1289                         return (rc);
1290                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1291                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1292                 end_synchronized_op(sc, LOCK_HELD);
1293                 break;
1294
1295         case SIOCSIFCAP:
1296                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1297                 if (rc)
1298                         return (rc);
1299
1300                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1301                 if (mask & IFCAP_TXCSUM) {
1302                         ifp->if_capenable ^= IFCAP_TXCSUM;
1303                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1304
1305                         if (IFCAP_TSO4 & ifp->if_capenable &&
1306                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1307                                 ifp->if_capenable &= ~IFCAP_TSO4;
1308                                 if_printf(ifp,
1309                                     "tso4 disabled due to -txcsum.\n");
1310                         }
1311                 }
1312                 if (mask & IFCAP_TXCSUM_IPV6) {
1313                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1314                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1315
1316                         if (IFCAP_TSO6 & ifp->if_capenable &&
1317                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1318                                 ifp->if_capenable &= ~IFCAP_TSO6;
1319                                 if_printf(ifp,
1320                                     "tso6 disabled due to -txcsum6.\n");
1321                         }
1322                 }
1323                 if (mask & IFCAP_RXCSUM)
1324                         ifp->if_capenable ^= IFCAP_RXCSUM;
1325                 if (mask & IFCAP_RXCSUM_IPV6)
1326                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1327
1328                 /*
1329                  * Note that we leave CSUM_TSO alone (it is always set).  The
1330                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1331                  * sending a TSO request our way, so it's sufficient to toggle
1332                  * IFCAP_TSOx only.
1333                  */
1334                 if (mask & IFCAP_TSO4) {
1335                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1336                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1337                                 if_printf(ifp, "enable txcsum first.\n");
1338                                 rc = EAGAIN;
1339                                 goto fail;
1340                         }
1341                         ifp->if_capenable ^= IFCAP_TSO4;
1342                 }
1343                 if (mask & IFCAP_TSO6) {
1344                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1345                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1346                                 if_printf(ifp, "enable txcsum6 first.\n");
1347                                 rc = EAGAIN;
1348                                 goto fail;
1349                         }
1350                         ifp->if_capenable ^= IFCAP_TSO6;
1351                 }
1352                 if (mask & IFCAP_LRO) {
1353 #if defined(INET) || defined(INET6)
1354                         int i;
1355                         struct sge_rxq *rxq;
1356
1357                         ifp->if_capenable ^= IFCAP_LRO;
1358                         for_each_rxq(pi, i, rxq) {
1359                                 if (ifp->if_capenable & IFCAP_LRO)
1360                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1361                                 else
1362                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1363                         }
1364 #endif
1365                 }
1366 #ifdef TCP_OFFLOAD
1367                 if (mask & IFCAP_TOE) {
1368                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1369
1370                         rc = toe_capability(pi, enable);
1371                         if (rc != 0)
1372                                 goto fail;
1373
1374                         ifp->if_capenable ^= mask;
1375                 }
1376 #endif
1377                 if (mask & IFCAP_VLAN_HWTAGGING) {
1378                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1379                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1380                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1381                 }
1382                 if (mask & IFCAP_VLAN_MTU) {
1383                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1384
1385                         /* Need to find out how to disable auto-mtu-inflation */
1386                 }
1387                 if (mask & IFCAP_VLAN_HWTSO)
1388                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1389                 if (mask & IFCAP_VLAN_HWCSUM)
1390                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1391
1392 #ifdef VLAN_CAPABILITIES
1393                 VLAN_CAPABILITIES(ifp);
1394 #endif
1395 fail:
1396                 end_synchronized_op(sc, 0);
1397                 break;
1398
1399         case SIOCSIFMEDIA:
1400         case SIOCGIFMEDIA:
1401                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1402                 break;
1403
1404         case SIOCGI2C: {
1405                 struct ifi2creq i2c;
1406
1407                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1408                 if (rc != 0)
1409                         break;
1410                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1411                         rc = EPERM;
1412                         break;
1413                 }
1414                 if (i2c.len > sizeof(i2c.data)) {
1415                         rc = EINVAL;
1416                         break;
1417                 }
1418                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1419                 if (rc)
1420                         return (rc);
1421                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1422                     i2c.offset, i2c.len, &i2c.data[0]);
1423                 end_synchronized_op(sc, 0);
1424                 if (rc == 0)
1425                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1426                 break;
1427         }
1428
1429         default:
1430                 rc = ether_ioctl(ifp, cmd, data);
1431         }
1432
1433         return (rc);
1434 }
1435
1436 static int
1437 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1438 {
1439         struct port_info *pi = ifp->if_softc;
1440         struct adapter *sc = pi->adapter;
1441         struct sge_txq *txq;
1442         void *items[1];
1443         int rc;
1444
1445         M_ASSERTPKTHDR(m);
1446         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1447
1448         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1449                 m_freem(m);
1450                 return (ENETDOWN);
1451         }
1452
1453         rc = parse_pkt(&m);
1454         if (__predict_false(rc != 0)) {
1455                 MPASS(m == NULL);                       /* was freed already */
1456                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1457                 return (rc);
1458         }
1459
1460         /* Select a txq. */
1461         txq = &sc->sge.txq[pi->first_txq];
1462         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1463                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1464                     pi->rsrv_noflowq);
1465
1466         items[0] = m;
1467         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1468         if (__predict_false(rc != 0))
1469                 m_freem(m);
1470
1471         return (rc);
1472 }
1473
1474 static void
1475 cxgbe_qflush(struct ifnet *ifp)
1476 {
1477         struct port_info *pi = ifp->if_softc;
1478         struct sge_txq *txq;
1479         int i;
1480
1481         /* queues do not exist if !PORT_INIT_DONE. */
1482         if (pi->flags & PORT_INIT_DONE) {
1483                 for_each_txq(pi, i, txq) {
1484                         TXQ_LOCK(txq);
1485                         txq->eq.flags &= ~EQ_ENABLED;
1486                         TXQ_UNLOCK(txq);
1487                         while (!mp_ring_is_idle(txq->r)) {
1488                                 mp_ring_check_drainage(txq->r, 0);
1489                                 pause("qflush", 1);
1490                         }
1491                 }
1492         }
1493         if_qflush(ifp);
1494 }
1495
1496 static uint64_t
1497 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1498 {
1499         struct port_info *pi = ifp->if_softc;
1500         struct adapter *sc = pi->adapter;
1501         struct port_stats *s = &pi->stats;
1502
1503         cxgbe_refresh_stats(sc, pi);
1504
1505         switch (c) {
1506         case IFCOUNTER_IPACKETS:
1507                 return (s->rx_frames - s->rx_pause);
1508
1509         case IFCOUNTER_IERRORS:
1510                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1511                     s->rx_fcs_err + s->rx_len_err);
1512
1513         case IFCOUNTER_OPACKETS:
1514                 return (s->tx_frames - s->tx_pause);
1515
1516         case IFCOUNTER_OERRORS:
1517                 return (s->tx_error_frames);
1518
1519         case IFCOUNTER_IBYTES:
1520                 return (s->rx_octets - s->rx_pause * 64);
1521
1522         case IFCOUNTER_OBYTES:
1523                 return (s->tx_octets - s->tx_pause * 64);
1524
1525         case IFCOUNTER_IMCASTS:
1526                 return (s->rx_mcast_frames - s->rx_pause);
1527
1528         case IFCOUNTER_OMCASTS:
1529                 return (s->tx_mcast_frames - s->tx_pause);
1530
1531         case IFCOUNTER_IQDROPS:
1532                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1533                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1534                     s->rx_trunc3 + pi->tnl_cong_drops);
1535
1536         case IFCOUNTER_OQDROPS: {
1537                 uint64_t drops;
1538
1539                 drops = s->tx_drop;
1540                 if (pi->flags & PORT_INIT_DONE) {
1541                         int i;
1542                         struct sge_txq *txq;
1543
1544                         for_each_txq(pi, i, txq)
1545                                 drops += counter_u64_fetch(txq->r->drops);
1546                 }
1547
1548                 return (drops);
1549
1550         }
1551
1552         default:
1553                 return (if_get_counter_default(ifp, c));
1554         }
1555 }
1556
1557 static int
1558 cxgbe_media_change(struct ifnet *ifp)
1559 {
1560         struct port_info *pi = ifp->if_softc;
1561
1562         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1563
1564         return (EOPNOTSUPP);
1565 }
1566
1567 static void
1568 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1569 {
1570         struct port_info *pi = ifp->if_softc;
1571         struct ifmedia *media = NULL;
1572         struct ifmedia_entry *cur;
1573         int speed = pi->link_cfg.speed;
1574 #ifdef INVARIANTS
1575         int data = (pi->port_type << 8) | pi->mod_type;
1576 #endif
1577
1578         if (ifp == pi->ifp)
1579                 media = &pi->media;
1580 #ifdef DEV_NETMAP
1581         else if (ifp == pi->nm_ifp)
1582                 media = &pi->nm_media;
1583 #endif
1584         MPASS(media != NULL);
1585
1586         cur = media->ifm_cur;
1587         MPASS(cur->ifm_data == data);
1588
1589         ifmr->ifm_status = IFM_AVALID;
1590         if (!pi->link_cfg.link_ok)
1591                 return;
1592
1593         ifmr->ifm_status |= IFM_ACTIVE;
1594
1595         /* active and current will differ iff current media is autoselect. */
1596         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1597                 return;
1598
1599         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1600         if (speed == SPEED_10000)
1601                 ifmr->ifm_active |= IFM_10G_T;
1602         else if (speed == SPEED_1000)
1603                 ifmr->ifm_active |= IFM_1000_T;
1604         else if (speed == SPEED_100)
1605                 ifmr->ifm_active |= IFM_100_TX;
1606         else if (speed == SPEED_10)
1607                 ifmr->ifm_active |= IFM_10_T;
1608         else
1609                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1610                             speed));
1611 }
1612
1613 void
1614 t4_fatal_err(struct adapter *sc)
1615 {
1616         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1617         t4_intr_disable(sc);
1618         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1619             device_get_nameunit(sc->dev));
1620 }
1621
1622 static int
1623 map_bars_0_and_4(struct adapter *sc)
1624 {
1625         sc->regs_rid = PCIR_BAR(0);
1626         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1627             &sc->regs_rid, RF_ACTIVE);
1628         if (sc->regs_res == NULL) {
1629                 device_printf(sc->dev, "cannot map registers.\n");
1630                 return (ENXIO);
1631         }
1632         sc->bt = rman_get_bustag(sc->regs_res);
1633         sc->bh = rman_get_bushandle(sc->regs_res);
1634         sc->mmio_len = rman_get_size(sc->regs_res);
1635         setbit(&sc->doorbells, DOORBELL_KDB);
1636
1637         sc->msix_rid = PCIR_BAR(4);
1638         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1639             &sc->msix_rid, RF_ACTIVE);
1640         if (sc->msix_res == NULL) {
1641                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1642                 return (ENXIO);
1643         }
1644
1645         return (0);
1646 }
1647
1648 static int
1649 map_bar_2(struct adapter *sc)
1650 {
1651
1652         /*
1653          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1654          * to map it if RDMA is disabled.
1655          */
1656         if (is_t4(sc) && sc->rdmacaps == 0)
1657                 return (0);
1658
1659         sc->udbs_rid = PCIR_BAR(2);
1660         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1661             &sc->udbs_rid, RF_ACTIVE);
1662         if (sc->udbs_res == NULL) {
1663                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1664                 return (ENXIO);
1665         }
1666         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1667
1668         if (is_t5(sc)) {
1669                 setbit(&sc->doorbells, DOORBELL_UDB);
1670 #if defined(__i386__) || defined(__amd64__)
1671                 if (t5_write_combine) {
1672                         int rc;
1673
1674                         /*
1675                          * Enable write combining on BAR2.  This is the
1676                          * userspace doorbell BAR and is split into 128B
1677                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1678                          * with an egress queue.  The first 64B has the doorbell
1679                          * and the second 64B can be used to submit a tx work
1680                          * request with an implicit doorbell.
1681                          */
1682
1683                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1684                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1685                         if (rc == 0) {
1686                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1687                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1688                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1689                         } else {
1690                                 device_printf(sc->dev,
1691                                     "couldn't enable write combining: %d\n",
1692                                     rc);
1693                         }
1694
1695                         t4_write_reg(sc, A_SGE_STAT_CFG,
1696                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1697                 }
1698 #endif
1699         }
1700
1701         return (0);
1702 }
1703
1704 static const struct memwin t4_memwin[] = {
1705         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1706         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1707         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1708 };
1709
1710 static const struct memwin t5_memwin[] = {
1711         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1712         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1713         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1714 };
1715
1716 static void
1717 setup_memwin(struct adapter *sc)
1718 {
1719         const struct memwin *mw;
1720         int i, n;
1721         uint32_t bar0;
1722
1723         if (is_t4(sc)) {
1724                 /*
1725                  * Read low 32b of bar0 indirectly via the hardware backdoor
1726                  * mechanism.  Works from within PCI passthrough environments
1727                  * too, where rman_get_start() can return a different value.  We
1728                  * need to program the T4 memory window decoders with the actual
1729                  * addresses that will be coming across the PCIe link.
1730                  */
1731                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1732                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1733
1734                 mw = &t4_memwin[0];
1735                 n = nitems(t4_memwin);
1736         } else {
1737                 /* T5 uses the relative offset inside the PCIe BAR */
1738                 bar0 = 0;
1739
1740                 mw = &t5_memwin[0];
1741                 n = nitems(t5_memwin);
1742         }
1743
1744         for (i = 0; i < n; i++, mw++) {
1745                 t4_write_reg(sc,
1746                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1747                     (mw->base + bar0) | V_BIR(0) |
1748                     V_WINDOW(ilog2(mw->aperture) - 10));
1749         }
1750
1751         /* flush */
1752         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1753 }
1754
1755 /*
1756  * Verify that the memory range specified by the addr/len pair is valid and lies
1757  * entirely within a single region (EDCx or MCx).
1758  */
1759 static int
1760 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1761 {
1762         uint32_t em, addr_len, maddr, mlen;
1763
1764         /* Memory can only be accessed in naturally aligned 4 byte units */
1765         if (addr & 3 || len & 3 || len == 0)
1766                 return (EINVAL);
1767
1768         /* Enabled memories */
1769         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1770         if (em & F_EDRAM0_ENABLE) {
1771                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1772                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1773                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1774                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1775                     addr + len <= maddr + mlen)
1776                         return (0);
1777         }
1778         if (em & F_EDRAM1_ENABLE) {
1779                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1780                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1781                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1782                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1783                     addr + len <= maddr + mlen)
1784                         return (0);
1785         }
1786         if (em & F_EXT_MEM_ENABLE) {
1787                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1788                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1789                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1790                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1791                     addr + len <= maddr + mlen)
1792                         return (0);
1793         }
1794         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1795                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1796                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1797                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1798                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1799                     addr + len <= maddr + mlen)
1800                         return (0);
1801         }
1802
1803         return (EFAULT);
1804 }
1805
1806 static int
1807 fwmtype_to_hwmtype(int mtype)
1808 {
1809
1810         switch (mtype) {
1811         case FW_MEMTYPE_EDC0:
1812                 return (MEM_EDC0);
1813         case FW_MEMTYPE_EDC1:
1814                 return (MEM_EDC1);
1815         case FW_MEMTYPE_EXTMEM:
1816                 return (MEM_MC0);
1817         case FW_MEMTYPE_EXTMEM1:
1818                 return (MEM_MC1);
1819         default:
1820                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1821         }
1822 }
1823
1824 /*
1825  * Verify that the memory range specified by the memtype/offset/len pair is
1826  * valid and lies entirely within the memtype specified.  The global address of
1827  * the start of the range is returned in addr.
1828  */
1829 static int
1830 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1831     uint32_t *addr)
1832 {
1833         uint32_t em, addr_len, maddr, mlen;
1834
1835         /* Memory can only be accessed in naturally aligned 4 byte units */
1836         if (off & 3 || len & 3 || len == 0)
1837                 return (EINVAL);
1838
1839         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1840         switch (fwmtype_to_hwmtype(mtype)) {
1841         case MEM_EDC0:
1842                 if (!(em & F_EDRAM0_ENABLE))
1843                         return (EINVAL);
1844                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1845                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1846                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1847                 break;
1848         case MEM_EDC1:
1849                 if (!(em & F_EDRAM1_ENABLE))
1850                         return (EINVAL);
1851                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1852                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1853                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1854                 break;
1855         case MEM_MC:
1856                 if (!(em & F_EXT_MEM_ENABLE))
1857                         return (EINVAL);
1858                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1859                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1860                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1861                 break;
1862         case MEM_MC1:
1863                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1864                         return (EINVAL);
1865                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1866                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1867                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1868                 break;
1869         default:
1870                 return (EINVAL);
1871         }
1872
1873         if (mlen > 0 && off < mlen && off + len <= mlen) {
1874                 *addr = maddr + off;    /* global address */
1875                 return (0);
1876         }
1877
1878         return (EFAULT);
1879 }
1880
1881 static void
1882 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1883 {
1884         const struct memwin *mw;
1885
1886         if (is_t4(sc)) {
1887                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1888                     ("%s: incorrect memwin# (%d)", __func__, win));
1889                 mw = &t4_memwin[win];
1890         } else {
1891                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1892                     ("%s: incorrect memwin# (%d)", __func__, win));
1893                 mw = &t5_memwin[win];
1894         }
1895
1896         if (base != NULL)
1897                 *base = mw->base;
1898         if (aperture != NULL)
1899                 *aperture = mw->aperture;
1900 }
1901
1902 /*
1903  * Positions the memory window such that it can be used to access the specified
1904  * address in the chip's address space.  The return value is the offset of addr
1905  * from the start of the window.
1906  */
1907 static uint32_t
1908 position_memwin(struct adapter *sc, int n, uint32_t addr)
1909 {
1910         uint32_t start, pf;
1911         uint32_t reg;
1912
1913         KASSERT(n >= 0 && n <= 3,
1914             ("%s: invalid window %d.", __func__, n));
1915         KASSERT((addr & 3) == 0,
1916             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1917
1918         if (is_t4(sc)) {
1919                 pf = 0;
1920                 start = addr & ~0xf;    /* start must be 16B aligned */
1921         } else {
1922                 pf = V_PFNUM(sc->pf);
1923                 start = addr & ~0x7f;   /* start must be 128B aligned */
1924         }
1925         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1926
1927         t4_write_reg(sc, reg, start | pf);
1928         t4_read_reg(sc, reg);
1929
1930         return (addr - start);
1931 }
1932
1933 static int
1934 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1935     struct intrs_and_queues *iaq)
1936 {
1937         int rc, itype, navail, nrxq10g, nrxq1g, n;
1938         int nofldrxq10g = 0, nofldrxq1g = 0;
1939         int nnmrxq10g = 0, nnmrxq1g = 0;
1940
1941         bzero(iaq, sizeof(*iaq));
1942
1943         iaq->ntxq10g = t4_ntxq10g;
1944         iaq->ntxq1g = t4_ntxq1g;
1945         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1946         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1947         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1948 #ifdef TCP_OFFLOAD
1949         if (is_offload(sc)) {
1950                 iaq->nofldtxq10g = t4_nofldtxq10g;
1951                 iaq->nofldtxq1g = t4_nofldtxq1g;
1952                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1953                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1954         }
1955 #endif
1956 #ifdef DEV_NETMAP
1957         iaq->nnmtxq10g = t4_nnmtxq10g;
1958         iaq->nnmtxq1g = t4_nnmtxq1g;
1959         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1960         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1961 #endif
1962
1963         for (itype = INTR_MSIX; itype; itype >>= 1) {
1964
1965                 if ((itype & t4_intr_types) == 0)
1966                         continue;       /* not allowed */
1967
1968                 if (itype == INTR_MSIX)
1969                         navail = pci_msix_count(sc->dev);
1970                 else if (itype == INTR_MSI)
1971                         navail = pci_msi_count(sc->dev);
1972                 else
1973                         navail = 1;
1974 restart:
1975                 if (navail == 0)
1976                         continue;
1977
1978                 iaq->intr_type = itype;
1979                 iaq->intr_flags_10g = 0;
1980                 iaq->intr_flags_1g = 0;
1981
1982                 /*
1983                  * Best option: an interrupt vector for errors, one for the
1984                  * firmware event queue, and one for every rxq (NIC, TOE, and
1985                  * netmap).
1986                  */
1987                 iaq->nirq = T4_EXTRA_INTR;
1988                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1989                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1990                 if (iaq->nirq <= navail &&
1991                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1992                         iaq->intr_flags_10g = INTR_ALL;
1993                         iaq->intr_flags_1g = INTR_ALL;
1994                         goto allocate;
1995                 }
1996
1997                 /*
1998                  * Second best option: a vector for errors, one for the firmware
1999                  * event queue, and vectors for either all the NIC rx queues or
2000                  * all the TOE rx queues.  The queues that don't get vectors
2001                  * will forward their interrupts to those that do.
2002                  *
2003                  * Note: netmap rx queues cannot be created early and so they
2004                  * can't be setup to receive forwarded interrupts for others.
2005                  */
2006                 iaq->nirq = T4_EXTRA_INTR;
2007                 if (nrxq10g >= nofldrxq10g) {
2008                         iaq->intr_flags_10g = INTR_RXQ;
2009                         iaq->nirq += n10g * nrxq10g;
2010 #ifdef DEV_NETMAP
2011                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2012 #endif
2013                 } else {
2014                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2015                         iaq->nirq += n10g * nofldrxq10g;
2016 #ifdef DEV_NETMAP
2017                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2018 #endif
2019                 }
2020                 if (nrxq1g >= nofldrxq1g) {
2021                         iaq->intr_flags_1g = INTR_RXQ;
2022                         iaq->nirq += n1g * nrxq1g;
2023 #ifdef DEV_NETMAP
2024                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2025 #endif
2026                 } else {
2027                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2028                         iaq->nirq += n1g * nofldrxq1g;
2029 #ifdef DEV_NETMAP
2030                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2031 #endif
2032                 }
2033                 if (iaq->nirq <= navail &&
2034                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2035                         goto allocate;
2036
2037                 /*
2038                  * Next best option: an interrupt vector for errors, one for the
2039                  * firmware event queue, and at least one per port.  At this
2040                  * point we know we'll have to downsize nrxq and/or nofldrxq
2041                  * and/or nnmrxq to fit what's available to us.
2042                  */
2043                 iaq->nirq = T4_EXTRA_INTR;
2044                 iaq->nirq += n10g + n1g;
2045                 if (iaq->nirq <= navail) {
2046                         int leftover = navail - iaq->nirq;
2047
2048                         if (n10g > 0) {
2049                                 int target = max(nrxq10g, nofldrxq10g);
2050
2051                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2052                                     INTR_RXQ : INTR_OFLD_RXQ;
2053
2054                                 n = 1;
2055                                 while (n < target && leftover >= n10g) {
2056                                         leftover -= n10g;
2057                                         iaq->nirq += n10g;
2058                                         n++;
2059                                 }
2060                                 iaq->nrxq10g = min(n, nrxq10g);
2061 #ifdef TCP_OFFLOAD
2062                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2063 #endif
2064 #ifdef DEV_NETMAP
2065                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2066 #endif
2067                         }
2068
2069                         if (n1g > 0) {
2070                                 int target = max(nrxq1g, nofldrxq1g);
2071
2072                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2073                                     INTR_RXQ : INTR_OFLD_RXQ;
2074
2075                                 n = 1;
2076                                 while (n < target && leftover >= n1g) {
2077                                         leftover -= n1g;
2078                                         iaq->nirq += n1g;
2079                                         n++;
2080                                 }
2081                                 iaq->nrxq1g = min(n, nrxq1g);
2082 #ifdef TCP_OFFLOAD
2083                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2084 #endif
2085 #ifdef DEV_NETMAP
2086                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2087 #endif
2088                         }
2089
2090                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2091                                 goto allocate;
2092                 }
2093
2094                 /*
2095                  * Least desirable option: one interrupt vector for everything.
2096                  */
2097                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2098                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2099 #ifdef TCP_OFFLOAD
2100                 if (is_offload(sc))
2101                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2102 #endif
2103 #ifdef DEV_NETMAP
2104                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2105 #endif
2106
2107 allocate:
2108                 navail = iaq->nirq;
2109                 rc = 0;
2110                 if (itype == INTR_MSIX)
2111                         rc = pci_alloc_msix(sc->dev, &navail);
2112                 else if (itype == INTR_MSI)
2113                         rc = pci_alloc_msi(sc->dev, &navail);
2114
2115                 if (rc == 0) {
2116                         if (navail == iaq->nirq)
2117                                 return (0);
2118
2119                         /*
2120                          * Didn't get the number requested.  Use whatever number
2121                          * the kernel is willing to allocate (it's in navail).
2122                          */
2123                         device_printf(sc->dev, "fewer vectors than requested, "
2124                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2125                             itype, iaq->nirq, navail);
2126                         pci_release_msi(sc->dev);
2127                         goto restart;
2128                 }
2129
2130                 device_printf(sc->dev,
2131                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2132                     itype, rc, iaq->nirq, navail);
2133         }
2134
2135         device_printf(sc->dev,
2136             "failed to find a usable interrupt type.  "
2137             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2138             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2139
2140         return (ENXIO);
2141 }
2142
2143 #define FW_VERSION(chip) ( \
2144     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2145     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2146     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2147     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2148 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2149
2150 struct fw_info {
2151         uint8_t chip;
2152         char *kld_name;
2153         char *fw_mod_name;
2154         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2155 } fw_info[] = {
2156         {
2157                 .chip = CHELSIO_T4,
2158                 .kld_name = "t4fw_cfg",
2159                 .fw_mod_name = "t4fw",
2160                 .fw_hdr = {
2161                         .chip = FW_HDR_CHIP_T4,
2162                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2163                         .intfver_nic = FW_INTFVER(T4, NIC),
2164                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2165                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2166                         .intfver_ri = FW_INTFVER(T4, RI),
2167                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2168                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2169                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2170                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2171                 },
2172         }, {
2173                 .chip = CHELSIO_T5,
2174                 .kld_name = "t5fw_cfg",
2175                 .fw_mod_name = "t5fw",
2176                 .fw_hdr = {
2177                         .chip = FW_HDR_CHIP_T5,
2178                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2179                         .intfver_nic = FW_INTFVER(T5, NIC),
2180                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2181                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2182                         .intfver_ri = FW_INTFVER(T5, RI),
2183                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2184                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2185                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2186                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2187                 },
2188         }
2189 };
2190
2191 static struct fw_info *
2192 find_fw_info(int chip)
2193 {
2194         int i;
2195
2196         for (i = 0; i < nitems(fw_info); i++) {
2197                 if (fw_info[i].chip == chip)
2198                         return (&fw_info[i]);
2199         }
2200         return (NULL);
2201 }
2202
2203 /*
2204  * Is the given firmware API compatible with the one the driver was compiled
2205  * with?
2206  */
2207 static int
2208 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2209 {
2210
2211         /* short circuit if it's the exact same firmware version */
2212         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2213                 return (1);
2214
2215         /*
2216          * XXX: Is this too conservative?  Perhaps I should limit this to the
2217          * features that are supported in the driver.
2218          */
2219 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2220         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2221             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2222             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2223                 return (1);
2224 #undef SAME_INTF
2225
2226         return (0);
2227 }
2228
2229 /*
2230  * The firmware in the KLD is usable, but should it be installed?  This routine
2231  * explains itself in detail if it indicates the KLD firmware should be
2232  * installed.
2233  */
2234 static int
2235 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2236 {
2237         const char *reason;
2238
2239         if (!card_fw_usable) {
2240                 reason = "incompatible or unusable";
2241                 goto install;
2242         }
2243
2244         if (k > c) {
2245                 reason = "older than the version bundled with this driver";
2246                 goto install;
2247         }
2248
2249         if (t4_fw_install == 2 && k != c) {
2250                 reason = "different than the version bundled with this driver";
2251                 goto install;
2252         }
2253
2254         return (0);
2255
2256 install:
2257         if (t4_fw_install == 0) {
2258                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2259                     "but the driver is prohibited from installing a different "
2260                     "firmware on the card.\n",
2261                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2262                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2263
2264                 return (0);
2265         }
2266
2267         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2268             "installing firmware %u.%u.%u.%u on card.\n",
2269             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2270             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2271             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2272             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2273
2274         return (1);
2275 }
2276 /*
2277  * Establish contact with the firmware and determine if we are the master driver
2278  * or not, and whether we are responsible for chip initialization.
2279  */
2280 static int
2281 prep_firmware(struct adapter *sc)
2282 {
2283         const struct firmware *fw = NULL, *default_cfg;
2284         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2285         enum dev_state state;
2286         struct fw_info *fw_info;
2287         struct fw_hdr *card_fw;         /* fw on the card */
2288         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2289         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2290                                            against */
2291
2292         /* Contact firmware. */
2293         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2294         if (rc < 0 || state == DEV_STATE_ERR) {
2295                 rc = -rc;
2296                 device_printf(sc->dev,
2297                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2298                 return (rc);
2299         }
2300         pf = rc;
2301         if (pf == sc->mbox)
2302                 sc->flags |= MASTER_PF;
2303         else if (state == DEV_STATE_UNINIT) {
2304                 /*
2305                  * We didn't get to be the master so we definitely won't be
2306                  * configuring the chip.  It's a bug if someone else hasn't
2307                  * configured it already.
2308                  */
2309                 device_printf(sc->dev, "couldn't be master(%d), "
2310                     "device not already initialized either(%d).\n", rc, state);
2311                 return (EDOOFUS);
2312         }
2313
2314         /* This is the firmware whose headers the driver was compiled against */
2315         fw_info = find_fw_info(chip_id(sc));
2316         if (fw_info == NULL) {
2317                 device_printf(sc->dev,
2318                     "unable to look up firmware information for chip %d.\n",
2319                     chip_id(sc));
2320                 return (EINVAL);
2321         }
2322         drv_fw = &fw_info->fw_hdr;
2323
2324         /*
2325          * The firmware KLD contains many modules.  The KLD name is also the
2326          * name of the module that contains the default config file.
2327          */
2328         default_cfg = firmware_get(fw_info->kld_name);
2329
2330         /* Read the header of the firmware on the card */
2331         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2332         rc = -t4_read_flash(sc, FLASH_FW_START,
2333             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2334         if (rc == 0)
2335                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2336         else {
2337                 device_printf(sc->dev,
2338                     "Unable to read card's firmware header: %d\n", rc);
2339                 card_fw_usable = 0;
2340         }
2341
2342         /* This is the firmware in the KLD */
2343         fw = firmware_get(fw_info->fw_mod_name);
2344         if (fw != NULL) {
2345                 kld_fw = (const void *)fw->data;
2346                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2347         } else {
2348                 kld_fw = NULL;
2349                 kld_fw_usable = 0;
2350         }
2351
2352         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2353             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2354                 /*
2355                  * Common case: the firmware on the card is an exact match and
2356                  * the KLD is an exact match too, or the KLD is
2357                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2358                  * here -- use cxgbetool loadfw if you want to reinstall the
2359                  * same firmware as the one on the card.
2360                  */
2361         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2362             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2363             be32toh(card_fw->fw_ver))) {
2364
2365                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2366                 if (rc != 0) {
2367                         device_printf(sc->dev,
2368                             "failed to install firmware: %d\n", rc);
2369                         goto done;
2370                 }
2371
2372                 /* Installed successfully, update the cached header too. */
2373                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2374                 card_fw_usable = 1;
2375                 need_fw_reset = 0;      /* already reset as part of load_fw */
2376         }
2377
2378         if (!card_fw_usable) {
2379                 uint32_t d, c, k;
2380
2381                 d = ntohl(drv_fw->fw_ver);
2382                 c = ntohl(card_fw->fw_ver);
2383                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2384
2385                 device_printf(sc->dev, "Cannot find a usable firmware: "
2386                     "fw_install %d, chip state %d, "
2387                     "driver compiled with %d.%d.%d.%d, "
2388                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2389                     t4_fw_install, state,
2390                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2391                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2392                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2393                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2394                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2395                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2396                 rc = EINVAL;
2397                 goto done;
2398         }
2399
2400         /* We're using whatever's on the card and it's known to be good. */
2401         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2402         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2403             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2404             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2405             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2406             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2407         t4_get_tp_version(sc, &sc->params.tp_vers);
2408
2409         /* Reset device */
2410         if (need_fw_reset &&
2411             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2412                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2413                 if (rc != ETIMEDOUT && rc != EIO)
2414                         t4_fw_bye(sc, sc->mbox);
2415                 goto done;
2416         }
2417         sc->flags |= FW_OK;
2418
2419         rc = get_params__pre_init(sc);
2420         if (rc != 0)
2421                 goto done; /* error message displayed already */
2422
2423         /* Partition adapter resources as specified in the config file. */
2424         if (state == DEV_STATE_UNINIT) {
2425
2426                 KASSERT(sc->flags & MASTER_PF,
2427                     ("%s: trying to change chip settings when not master.",
2428                     __func__));
2429
2430                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2431                 if (rc != 0)
2432                         goto done;      /* error message displayed already */
2433
2434                 t4_tweak_chip_settings(sc);
2435
2436                 /* get basic stuff going */
2437                 rc = -t4_fw_initialize(sc, sc->mbox);
2438                 if (rc != 0) {
2439                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2440                         goto done;
2441                 }
2442         } else {
2443                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2444                 sc->cfcsum = 0;
2445         }
2446
2447 done:
2448         free(card_fw, M_CXGBE);
2449         if (fw != NULL)
2450                 firmware_put(fw, FIRMWARE_UNLOAD);
2451         if (default_cfg != NULL)
2452                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2453
2454         return (rc);
2455 }
2456
2457 #define FW_PARAM_DEV(param) \
2458         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2459          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2460 #define FW_PARAM_PFVF(param) \
2461         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2462          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2463
2464 /*
2465  * Partition chip resources for use between various PFs, VFs, etc.
2466  */
2467 static int
2468 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2469     const char *name_prefix)
2470 {
2471         const struct firmware *cfg = NULL;
2472         int rc = 0;
2473         struct fw_caps_config_cmd caps;
2474         uint32_t mtype, moff, finicsum, cfcsum;
2475
2476         /*
2477          * Figure out what configuration file to use.  Pick the default config
2478          * file for the card if the user hasn't specified one explicitly.
2479          */
2480         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2481         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2482                 /* Card specific overrides go here. */
2483                 if (pci_get_device(sc->dev) == 0x440a)
2484                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2485                 if (is_fpga(sc))
2486                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2487         }
2488
2489         /*
2490          * We need to load another module if the profile is anything except
2491          * "default" or "flash".
2492          */
2493         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2494             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2495                 char s[32];
2496
2497                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2498                 cfg = firmware_get(s);
2499                 if (cfg == NULL) {
2500                         if (default_cfg != NULL) {
2501                                 device_printf(sc->dev,
2502                                     "unable to load module \"%s\" for "
2503                                     "configuration profile \"%s\", will use "
2504                                     "the default config file instead.\n",
2505                                     s, sc->cfg_file);
2506                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2507                                     "%s", DEFAULT_CF);
2508                         } else {
2509                                 device_printf(sc->dev,
2510                                     "unable to load module \"%s\" for "
2511                                     "configuration profile \"%s\", will use "
2512                                     "the config file on the card's flash "
2513                                     "instead.\n", s, sc->cfg_file);
2514                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2515                                     "%s", FLASH_CF);
2516                         }
2517                 }
2518         }
2519
2520         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2521             default_cfg == NULL) {
2522                 device_printf(sc->dev,
2523                     "default config file not available, will use the config "
2524                     "file on the card's flash instead.\n");
2525                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2526         }
2527
2528         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2529                 u_int cflen, i, n;
2530                 const uint32_t *cfdata;
2531                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2532
2533                 KASSERT(cfg != NULL || default_cfg != NULL,
2534                     ("%s: no config to upload", __func__));
2535
2536                 /*
2537                  * Ask the firmware where it wants us to upload the config file.
2538                  */
2539                 param = FW_PARAM_DEV(CF);
2540                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2541                 if (rc != 0) {
2542                         /* No support for config file?  Shouldn't happen. */
2543                         device_printf(sc->dev,
2544                             "failed to query config file location: %d.\n", rc);
2545                         goto done;
2546                 }
2547                 mtype = G_FW_PARAMS_PARAM_Y(val);
2548                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2549
2550                 /*
2551                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2552                  * useless stuffing/comments at the end of the config file so
2553                  * it's ok to simply throw away the last remaining bytes when
2554                  * the config file is not an exact multiple of 4.  This also
2555                  * helps with the validate_mt_off_len check.
2556                  */
2557                 if (cfg != NULL) {
2558                         cflen = cfg->datasize & ~3;
2559                         cfdata = cfg->data;
2560                 } else {
2561                         cflen = default_cfg->datasize & ~3;
2562                         cfdata = default_cfg->data;
2563                 }
2564
2565                 if (cflen > FLASH_CFG_MAX_SIZE) {
2566                         device_printf(sc->dev,
2567                             "config file too long (%d, max allowed is %d).  "
2568                             "Will try to use the config on the card, if any.\n",
2569                             cflen, FLASH_CFG_MAX_SIZE);
2570                         goto use_config_on_flash;
2571                 }
2572
2573                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2574                 if (rc != 0) {
2575                         device_printf(sc->dev,
2576                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2577                             "Will try to use the config on the card, if any.\n",
2578                             __func__, mtype, moff, cflen, rc);
2579                         goto use_config_on_flash;
2580                 }
2581
2582                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2583                 while (cflen) {
2584                         off = position_memwin(sc, 2, addr);
2585                         n = min(cflen, mw_aperture - off);
2586                         for (i = 0; i < n; i += 4)
2587                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2588                         cflen -= n;
2589                         addr += n;
2590                 }
2591         } else {
2592 use_config_on_flash:
2593                 mtype = FW_MEMTYPE_FLASH;
2594                 moff = t4_flash_cfg_addr(sc);
2595         }
2596
2597         bzero(&caps, sizeof(caps));
2598         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2599             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2600         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2601             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2602             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2603         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2604         if (rc != 0) {
2605                 device_printf(sc->dev,
2606                     "failed to pre-process config file: %d "
2607                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2608                 goto done;
2609         }
2610
2611         finicsum = be32toh(caps.finicsum);
2612         cfcsum = be32toh(caps.cfcsum);
2613         if (finicsum != cfcsum) {
2614                 device_printf(sc->dev,
2615                     "WARNING: config file checksum mismatch: %08x %08x\n",
2616                     finicsum, cfcsum);
2617         }
2618         sc->cfcsum = cfcsum;
2619
2620 #define LIMIT_CAPS(x) do { \
2621         caps.x &= htobe16(t4_##x##_allowed); \
2622 } while (0)
2623
2624         /*
2625          * Let the firmware know what features will (not) be used so it can tune
2626          * things accordingly.
2627          */
2628         LIMIT_CAPS(linkcaps);
2629         LIMIT_CAPS(niccaps);
2630         LIMIT_CAPS(toecaps);
2631         LIMIT_CAPS(rdmacaps);
2632         LIMIT_CAPS(iscsicaps);
2633         LIMIT_CAPS(fcoecaps);
2634 #undef LIMIT_CAPS
2635
2636         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2637             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2638         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2639         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2640         if (rc != 0) {
2641                 device_printf(sc->dev,
2642                     "failed to process config file: %d.\n", rc);
2643         }
2644 done:
2645         if (cfg != NULL)
2646                 firmware_put(cfg, FIRMWARE_UNLOAD);
2647         return (rc);
2648 }
2649
2650 /*
2651  * Retrieve parameters that are needed (or nice to have) very early.
2652  */
2653 static int
2654 get_params__pre_init(struct adapter *sc)
2655 {
2656         int rc;
2657         uint32_t param[2], val[2];
2658         struct fw_devlog_cmd cmd;
2659         struct devlog_params *dlog = &sc->params.devlog;
2660
2661         param[0] = FW_PARAM_DEV(PORTVEC);
2662         param[1] = FW_PARAM_DEV(CCLK);
2663         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2664         if (rc != 0) {
2665                 device_printf(sc->dev,
2666                     "failed to query parameters (pre_init): %d.\n", rc);
2667                 return (rc);
2668         }
2669
2670         sc->params.portvec = val[0];
2671         sc->params.nports = bitcount32(val[0]);
2672         sc->params.vpd.cclk = val[1];
2673
2674         /* Read device log parameters. */
2675         bzero(&cmd, sizeof(cmd));
2676         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2677             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2678         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2679         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2680         if (rc != 0) {
2681                 device_printf(sc->dev,
2682                     "failed to get devlog parameters: %d.\n", rc);
2683                 bzero(dlog, sizeof (*dlog));
2684                 rc = 0; /* devlog isn't critical for device operation */
2685         } else {
2686                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2687                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2688                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2689                 dlog->size = be32toh(cmd.memsize_devlog);
2690         }
2691
2692         return (rc);
2693 }
2694
2695 /*
2696  * Retrieve various parameters that are of interest to the driver.  The device
2697  * has been initialized by the firmware at this point.
2698  */
2699 static int
2700 get_params__post_init(struct adapter *sc)
2701 {
2702         int rc;
2703         uint32_t param[7], val[7];
2704         struct fw_caps_config_cmd caps;
2705
2706         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2707         param[1] = FW_PARAM_PFVF(EQ_START);
2708         param[2] = FW_PARAM_PFVF(FILTER_START);
2709         param[3] = FW_PARAM_PFVF(FILTER_END);
2710         param[4] = FW_PARAM_PFVF(L2T_START);
2711         param[5] = FW_PARAM_PFVF(L2T_END);
2712         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2713         if (rc != 0) {
2714                 device_printf(sc->dev,
2715                     "failed to query parameters (post_init): %d.\n", rc);
2716                 return (rc);
2717         }
2718
2719         sc->sge.iq_start = val[0];
2720         sc->sge.eq_start = val[1];
2721         sc->tids.ftid_base = val[2];
2722         sc->tids.nftids = val[3] - val[2] + 1;
2723         sc->params.ftid_min = val[2];
2724         sc->params.ftid_max = val[3];
2725         sc->vres.l2t.start = val[4];
2726         sc->vres.l2t.size = val[5] - val[4] + 1;
2727         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2728             ("%s: L2 table size (%u) larger than expected (%u)",
2729             __func__, sc->vres.l2t.size, L2T_SIZE));
2730
2731         /* get capabilites */
2732         bzero(&caps, sizeof(caps));
2733         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2734             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2735         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2736         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2737         if (rc != 0) {
2738                 device_printf(sc->dev,
2739                     "failed to get card capabilities: %d.\n", rc);
2740                 return (rc);
2741         }
2742
2743 #define READ_CAPS(x) do { \
2744         sc->x = htobe16(caps.x); \
2745 } while (0)
2746         READ_CAPS(linkcaps);
2747         READ_CAPS(niccaps);
2748         READ_CAPS(toecaps);
2749         READ_CAPS(rdmacaps);
2750         READ_CAPS(iscsicaps);
2751         READ_CAPS(fcoecaps);
2752
2753         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2754                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2755                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2756                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2757                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2758                 if (rc != 0) {
2759                         device_printf(sc->dev,
2760                             "failed to query NIC parameters: %d.\n", rc);
2761                         return (rc);
2762                 }
2763                 sc->tids.etid_base = val[0];
2764                 sc->params.etid_min = val[0];
2765                 sc->tids.netids = val[1] - val[0] + 1;
2766                 sc->params.netids = sc->tids.netids;
2767                 sc->params.eo_wr_cred = val[2];
2768                 sc->params.ethoffload = 1;
2769         }
2770
2771         if (sc->toecaps) {
2772                 /* query offload-related parameters */
2773                 param[0] = FW_PARAM_DEV(NTID);
2774                 param[1] = FW_PARAM_PFVF(SERVER_START);
2775                 param[2] = FW_PARAM_PFVF(SERVER_END);
2776                 param[3] = FW_PARAM_PFVF(TDDP_START);
2777                 param[4] = FW_PARAM_PFVF(TDDP_END);
2778                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2779                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2780                 if (rc != 0) {
2781                         device_printf(sc->dev,
2782                             "failed to query TOE parameters: %d.\n", rc);
2783                         return (rc);
2784                 }
2785                 sc->tids.ntids = val[0];
2786                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2787                 sc->tids.stid_base = val[1];
2788                 sc->tids.nstids = val[2] - val[1] + 1;
2789                 sc->vres.ddp.start = val[3];
2790                 sc->vres.ddp.size = val[4] - val[3] + 1;
2791                 sc->params.ofldq_wr_cred = val[5];
2792                 sc->params.offload = 1;
2793         }
2794         if (sc->rdmacaps) {
2795                 param[0] = FW_PARAM_PFVF(STAG_START);
2796                 param[1] = FW_PARAM_PFVF(STAG_END);
2797                 param[2] = FW_PARAM_PFVF(RQ_START);
2798                 param[3] = FW_PARAM_PFVF(RQ_END);
2799                 param[4] = FW_PARAM_PFVF(PBL_START);
2800                 param[5] = FW_PARAM_PFVF(PBL_END);
2801                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2802                 if (rc != 0) {
2803                         device_printf(sc->dev,
2804                             "failed to query RDMA parameters(1): %d.\n", rc);
2805                         return (rc);
2806                 }
2807                 sc->vres.stag.start = val[0];
2808                 sc->vres.stag.size = val[1] - val[0] + 1;
2809                 sc->vres.rq.start = val[2];
2810                 sc->vres.rq.size = val[3] - val[2] + 1;
2811                 sc->vres.pbl.start = val[4];
2812                 sc->vres.pbl.size = val[5] - val[4] + 1;
2813
2814                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2815                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2816                 param[2] = FW_PARAM_PFVF(CQ_START);
2817                 param[3] = FW_PARAM_PFVF(CQ_END);
2818                 param[4] = FW_PARAM_PFVF(OCQ_START);
2819                 param[5] = FW_PARAM_PFVF(OCQ_END);
2820                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2821                 if (rc != 0) {
2822                         device_printf(sc->dev,
2823                             "failed to query RDMA parameters(2): %d.\n", rc);
2824                         return (rc);
2825                 }
2826                 sc->vres.qp.start = val[0];
2827                 sc->vres.qp.size = val[1] - val[0] + 1;
2828                 sc->vres.cq.start = val[2];
2829                 sc->vres.cq.size = val[3] - val[2] + 1;
2830                 sc->vres.ocq.start = val[4];
2831                 sc->vres.ocq.size = val[5] - val[4] + 1;
2832         }
2833         if (sc->iscsicaps) {
2834                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2835                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2836                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2837                 if (rc != 0) {
2838                         device_printf(sc->dev,
2839                             "failed to query iSCSI parameters: %d.\n", rc);
2840                         return (rc);
2841                 }
2842                 sc->vres.iscsi.start = val[0];
2843                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2844         }
2845
2846         /*
2847          * We've got the params we wanted to query via the firmware.  Now grab
2848          * some others directly from the chip.
2849          */
2850         rc = t4_read_chip_settings(sc);
2851
2852         return (rc);
2853 }
2854
2855 static int
2856 set_params__post_init(struct adapter *sc)
2857 {
2858         uint32_t param, val;
2859
2860         /* ask for encapsulated CPLs */
2861         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2862         val = 1;
2863         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2864
2865         return (0);
2866 }
2867
2868 #undef FW_PARAM_PFVF
2869 #undef FW_PARAM_DEV
2870
2871 static void
2872 t4_set_desc(struct adapter *sc)
2873 {
2874         char buf[128];
2875         struct adapter_params *p = &sc->params;
2876
2877         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2878             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2879             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2880
2881         device_set_desc_copy(sc->dev, buf);
2882 }
2883
2884 static void
2885 build_medialist(struct port_info *pi, struct ifmedia *media)
2886 {
2887         int data, m;
2888
2889         PORT_LOCK(pi);
2890
2891         ifmedia_removeall(media);
2892
2893         m = IFM_ETHER | IFM_FDX;
2894         data = (pi->port_type << 8) | pi->mod_type;
2895
2896         switch(pi->port_type) {
2897         case FW_PORT_TYPE_BT_XFI:
2898                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2899                 break;
2900
2901         case FW_PORT_TYPE_BT_XAUI:
2902                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2903                 /* fall through */
2904
2905         case FW_PORT_TYPE_BT_SGMII:
2906                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2907                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2908                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2909                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2910                 break;
2911
2912         case FW_PORT_TYPE_CX4:
2913                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2914                 ifmedia_set(media, m | IFM_10G_CX4);
2915                 break;
2916
2917         case FW_PORT_TYPE_QSFP_10G:
2918         case FW_PORT_TYPE_SFP:
2919         case FW_PORT_TYPE_FIBER_XFI:
2920         case FW_PORT_TYPE_FIBER_XAUI:
2921                 switch (pi->mod_type) {
2922
2923                 case FW_PORT_MOD_TYPE_LR:
2924                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2925                         ifmedia_set(media, m | IFM_10G_LR);
2926                         break;
2927
2928                 case FW_PORT_MOD_TYPE_SR:
2929                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2930                         ifmedia_set(media, m | IFM_10G_SR);
2931                         break;
2932
2933                 case FW_PORT_MOD_TYPE_LRM:
2934                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2935                         ifmedia_set(media, m | IFM_10G_LRM);
2936                         break;
2937
2938                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2939                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2940                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2941                         ifmedia_set(media, m | IFM_10G_TWINAX);
2942                         break;
2943
2944                 case FW_PORT_MOD_TYPE_NONE:
2945                         m &= ~IFM_FDX;
2946                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2947                         ifmedia_set(media, m | IFM_NONE);
2948                         break;
2949
2950                 case FW_PORT_MOD_TYPE_NA:
2951                 case FW_PORT_MOD_TYPE_ER:
2952                 default:
2953                         device_printf(pi->dev,
2954                             "unknown port_type (%d), mod_type (%d)\n",
2955                             pi->port_type, pi->mod_type);
2956                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2957                         ifmedia_set(media, m | IFM_UNKNOWN);
2958                         break;
2959                 }
2960                 break;
2961
2962         case FW_PORT_TYPE_QSFP:
2963                 switch (pi->mod_type) {
2964
2965                 case FW_PORT_MOD_TYPE_LR:
2966                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2967                         ifmedia_set(media, m | IFM_40G_LR4);
2968                         break;
2969
2970                 case FW_PORT_MOD_TYPE_SR:
2971                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2972                         ifmedia_set(media, m | IFM_40G_SR4);
2973                         break;
2974
2975                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2976                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2977                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2978                         ifmedia_set(media, m | IFM_40G_CR4);
2979                         break;
2980
2981                 case FW_PORT_MOD_TYPE_NONE:
2982                         m &= ~IFM_FDX;
2983                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2984                         ifmedia_set(media, m | IFM_NONE);
2985                         break;
2986
2987                 default:
2988                         device_printf(pi->dev,
2989                             "unknown port_type (%d), mod_type (%d)\n",
2990                             pi->port_type, pi->mod_type);
2991                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2992                         ifmedia_set(media, m | IFM_UNKNOWN);
2993                         break;
2994                 }
2995                 break;
2996
2997         default:
2998                 device_printf(pi->dev,
2999                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3000                     pi->mod_type);
3001                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
3002                 ifmedia_set(media, m | IFM_UNKNOWN);
3003                 break;
3004         }
3005
3006         PORT_UNLOCK(pi);
3007 }
3008
3009 #define FW_MAC_EXACT_CHUNK      7
3010
3011 /*
3012  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3013  * indicates which parameters should be programmed (the rest are left alone).
3014  */
3015 int
3016 update_mac_settings(struct ifnet *ifp, int flags)
3017 {
3018         int rc = 0;
3019         struct port_info *pi = ifp->if_softc;
3020         struct adapter *sc = pi->adapter;
3021         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3022         uint16_t viid = 0xffff;
3023         int16_t *xact_addr_filt = NULL;
3024
3025         ASSERT_SYNCHRONIZED_OP(sc);
3026         KASSERT(flags, ("%s: not told what to update.", __func__));
3027
3028         if (ifp == pi->ifp) {
3029                 viid = pi->viid;
3030                 xact_addr_filt = &pi->xact_addr_filt;
3031         }
3032 #ifdef DEV_NETMAP
3033         else if (ifp == pi->nm_ifp) {
3034                 viid = pi->nm_viid;
3035                 xact_addr_filt = &pi->nm_xact_addr_filt;
3036         }
3037 #endif
3038         if (flags & XGMAC_MTU)
3039                 mtu = ifp->if_mtu;
3040
3041         if (flags & XGMAC_PROMISC)
3042                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3043
3044         if (flags & XGMAC_ALLMULTI)
3045                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3046
3047         if (flags & XGMAC_VLANEX)
3048                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3049
3050         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3051                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3052                     1, vlanex, false);
3053                 if (rc) {
3054                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3055                             rc);
3056                         return (rc);
3057                 }
3058         }
3059
3060         if (flags & XGMAC_UCADDR) {
3061                 uint8_t ucaddr[ETHER_ADDR_LEN];
3062
3063                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3064                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3065                     true, true);
3066                 if (rc < 0) {
3067                         rc = -rc;
3068                         if_printf(ifp, "change_mac failed: %d\n", rc);
3069                         return (rc);
3070                 } else {
3071                         *xact_addr_filt = rc;
3072                         rc = 0;
3073                 }
3074         }
3075
3076         if (flags & XGMAC_MCADDRS) {
3077                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3078                 int del = 1;
3079                 uint64_t hash = 0;
3080                 struct ifmultiaddr *ifma;
3081                 int i = 0, j;
3082
3083                 if_maddr_rlock(ifp);
3084                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3085                         if (ifma->ifma_addr->sa_family != AF_LINK)
3086                                 continue;
3087                         mcaddr[i] =
3088                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3089                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3090                         i++;
3091
3092                         if (i == FW_MAC_EXACT_CHUNK) {
3093                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3094                                     i, mcaddr, NULL, &hash, 0);
3095                                 if (rc < 0) {
3096                                         rc = -rc;
3097                                         for (j = 0; j < i; j++) {
3098                                                 if_printf(ifp,
3099                                                     "failed to add mc address"
3100                                                     " %02x:%02x:%02x:"
3101                                                     "%02x:%02x:%02x rc=%d\n",
3102                                                     mcaddr[j][0], mcaddr[j][1],
3103                                                     mcaddr[j][2], mcaddr[j][3],
3104                                                     mcaddr[j][4], mcaddr[j][5],
3105                                                     rc);
3106                                         }
3107                                         goto mcfail;
3108                                 }
3109                                 del = 0;
3110                                 i = 0;
3111                         }
3112                 }
3113                 if (i > 0) {
3114                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3115                             mcaddr, NULL, &hash, 0);
3116                         if (rc < 0) {
3117                                 rc = -rc;
3118                                 for (j = 0; j < i; j++) {
3119                                         if_printf(ifp,
3120                                             "failed to add mc address"
3121                                             " %02x:%02x:%02x:"
3122                                             "%02x:%02x:%02x rc=%d\n",
3123                                             mcaddr[j][0], mcaddr[j][1],
3124                                             mcaddr[j][2], mcaddr[j][3],
3125                                             mcaddr[j][4], mcaddr[j][5],
3126                                             rc);
3127                                 }
3128                                 goto mcfail;
3129                         }
3130                 }
3131
3132                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3133                 if (rc != 0)
3134                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3135 mcfail:
3136                 if_maddr_runlock(ifp);
3137         }
3138
3139         return (rc);
3140 }
3141
3142 int
3143 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3144     char *wmesg)
3145 {
3146         int rc, pri;
3147
3148 #ifdef WITNESS
3149         /* the caller thinks it's ok to sleep, but is it really? */
3150         if (flags & SLEEP_OK)
3151                 pause("t4slptst", 1);
3152 #endif
3153
3154         if (INTR_OK)
3155                 pri = PCATCH;
3156         else
3157                 pri = 0;
3158
3159         ADAPTER_LOCK(sc);
3160         for (;;) {
3161
3162                 if (pi && IS_DOOMED(pi)) {
3163                         rc = ENXIO;
3164                         goto done;
3165                 }
3166
3167                 if (!IS_BUSY(sc)) {
3168                         rc = 0;
3169                         break;
3170                 }
3171
3172                 if (!(flags & SLEEP_OK)) {
3173                         rc = EBUSY;
3174                         goto done;
3175                 }
3176
3177                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3178                         rc = EINTR;
3179                         goto done;
3180                 }
3181         }
3182
3183         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3184         SET_BUSY(sc);
3185 #ifdef INVARIANTS
3186         sc->last_op = wmesg;
3187         sc->last_op_thr = curthread;
3188 #endif
3189
3190 done:
3191         if (!(flags & HOLD_LOCK) || rc)
3192                 ADAPTER_UNLOCK(sc);
3193
3194         return (rc);
3195 }
3196
3197 void
3198 end_synchronized_op(struct adapter *sc, int flags)
3199 {
3200
3201         if (flags & LOCK_HELD)
3202                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3203         else
3204                 ADAPTER_LOCK(sc);
3205
3206         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3207         CLR_BUSY(sc);
3208         wakeup(&sc->flags);
3209         ADAPTER_UNLOCK(sc);
3210 }
3211
3212 static int
3213 cxgbe_init_synchronized(struct port_info *pi)
3214 {
3215         struct adapter *sc = pi->adapter;
3216         struct ifnet *ifp = pi->ifp;
3217         int rc = 0, i;
3218         struct sge_txq *txq;
3219
3220         ASSERT_SYNCHRONIZED_OP(sc);
3221
3222         if (isset(&sc->open_device_map, pi->port_id)) {
3223                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3224                     ("mismatch between open_device_map and if_drv_flags"));
3225                 return (0);     /* already running */
3226         }
3227
3228         if (!(sc->flags & FULL_INIT_DONE) &&
3229             ((rc = adapter_full_init(sc)) != 0))
3230                 return (rc);    /* error message displayed already */
3231
3232         if (!(pi->flags & PORT_INIT_DONE) &&
3233             ((rc = port_full_init(pi)) != 0))
3234                 return (rc); /* error message displayed already */
3235
3236         rc = update_mac_settings(ifp, XGMAC_ALL);
3237         if (rc)
3238                 goto done;      /* error message displayed already */
3239
3240         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3241         if (rc != 0) {
3242                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3243                 goto done;
3244         }
3245
3246         /*
3247          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3248          * if this changes.
3249          */
3250
3251         for_each_txq(pi, i, txq) {
3252                 TXQ_LOCK(txq);
3253                 txq->eq.flags |= EQ_ENABLED;
3254                 TXQ_UNLOCK(txq);
3255         }
3256
3257         /*
3258          * The first iq of the first port to come up is used for tracing.
3259          */
3260         if (sc->traceq < 0) {
3261                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3262                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3263                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3264                     V_QUEUENUMBER(sc->traceq));
3265                 pi->flags |= HAS_TRACEQ;
3266         }
3267
3268         /* all ok */
3269         setbit(&sc->open_device_map, pi->port_id);
3270         PORT_LOCK(pi);
3271         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3272         PORT_UNLOCK(pi);
3273
3274         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3275 done:
3276         if (rc != 0)
3277                 cxgbe_uninit_synchronized(pi);
3278
3279         return (rc);
3280 }
3281
3282 /*
3283  * Idempotent.
3284  */
3285 static int
3286 cxgbe_uninit_synchronized(struct port_info *pi)
3287 {
3288         struct adapter *sc = pi->adapter;
3289         struct ifnet *ifp = pi->ifp;
3290         int rc, i;
3291         struct sge_txq *txq;
3292
3293         ASSERT_SYNCHRONIZED_OP(sc);
3294
3295         if (!(pi->flags & PORT_INIT_DONE)) {
3296                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3297                     ("uninited port is running"));
3298                 return (0);
3299         }
3300
3301         /*
3302          * Disable the VI so that all its data in either direction is discarded
3303          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3304          * tick) intact as the TP can deliver negative advice or data that it's
3305          * holding in its RAM (for an offloaded connection) even after the VI is
3306          * disabled.
3307          */
3308         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3309         if (rc) {
3310                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3311                 return (rc);
3312         }
3313
3314         for_each_txq(pi, i, txq) {
3315                 TXQ_LOCK(txq);
3316                 txq->eq.flags &= ~EQ_ENABLED;
3317                 TXQ_UNLOCK(txq);
3318         }
3319
3320         clrbit(&sc->open_device_map, pi->port_id);
3321         PORT_LOCK(pi);
3322         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3323         PORT_UNLOCK(pi);
3324
3325         pi->link_cfg.link_ok = 0;
3326         pi->link_cfg.speed = 0;
3327         pi->linkdnrc = -1;
3328         t4_os_link_changed(sc, pi->port_id, 0, -1);
3329
3330         return (0);
3331 }
3332
3333 /*
3334  * It is ok for this function to fail midway and return right away.  t4_detach
3335  * will walk the entire sc->irq list and clean up whatever is valid.
3336  */
3337 static int
3338 setup_intr_handlers(struct adapter *sc)
3339 {
3340         int rc, rid, p, q;
3341         char s[8];
3342         struct irq *irq;
3343         struct port_info *pi;
3344         struct sge_rxq *rxq;
3345 #ifdef TCP_OFFLOAD
3346         struct sge_ofld_rxq *ofld_rxq;
3347 #endif
3348 #ifdef DEV_NETMAP
3349         struct sge_nm_rxq *nm_rxq;
3350 #endif
3351
3352         /*
3353          * Setup interrupts.
3354          */
3355         irq = &sc->irq[0];
3356         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3357         if (sc->intr_count == 1)
3358                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3359
3360         /* Multiple interrupts. */
3361         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3362             ("%s: too few intr.", __func__));
3363
3364         /* The first one is always error intr */
3365         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3366         if (rc != 0)
3367                 return (rc);
3368         irq++;
3369         rid++;
3370
3371         /* The second one is always the firmware event queue */
3372         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3373         if (rc != 0)
3374                 return (rc);
3375         irq++;
3376         rid++;
3377
3378         for_each_port(sc, p) {
3379                 pi = sc->port[p];
3380
3381                 if (pi->flags & INTR_RXQ) {
3382                         for_each_rxq(pi, q, rxq) {
3383                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3384                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3385                                     s);
3386                                 if (rc != 0)
3387                                         return (rc);
3388                                 irq++;
3389                                 rid++;
3390                         }
3391                 }
3392 #ifdef TCP_OFFLOAD
3393                 if (pi->flags & INTR_OFLD_RXQ) {
3394                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3395                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3396                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3397                                     ofld_rxq, s);
3398                                 if (rc != 0)
3399                                         return (rc);
3400                                 irq++;
3401                                 rid++;
3402                         }
3403                 }
3404 #endif
3405 #ifdef DEV_NETMAP
3406                 if (pi->flags & INTR_NM_RXQ) {
3407                         for_each_nm_rxq(pi, q, nm_rxq) {
3408                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3409                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3410                                     nm_rxq, s);
3411                                 if (rc != 0)
3412                                         return (rc);
3413                                 irq++;
3414                                 rid++;
3415                         }
3416                 }
3417 #endif
3418         }
3419         MPASS(irq == &sc->irq[sc->intr_count]);
3420
3421         return (0);
3422 }
3423
3424 int
3425 adapter_full_init(struct adapter *sc)
3426 {
3427         int rc, i;
3428
3429         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3430         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3431             ("%s: FULL_INIT_DONE already", __func__));
3432
3433         /*
3434          * queues that belong to the adapter (not any particular port).
3435          */
3436         rc = t4_setup_adapter_queues(sc);
3437         if (rc != 0)
3438                 goto done;
3439
3440         for (i = 0; i < nitems(sc->tq); i++) {
3441                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3442                     taskqueue_thread_enqueue, &sc->tq[i]);
3443                 if (sc->tq[i] == NULL) {
3444                         device_printf(sc->dev,
3445                             "failed to allocate task queue %d\n", i);
3446                         rc = ENOMEM;
3447                         goto done;
3448                 }
3449                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3450                     device_get_nameunit(sc->dev), i);
3451         }
3452
3453         t4_intr_enable(sc);
3454         sc->flags |= FULL_INIT_DONE;
3455 done:
3456         if (rc != 0)
3457                 adapter_full_uninit(sc);
3458
3459         return (rc);
3460 }
3461
3462 int
3463 adapter_full_uninit(struct adapter *sc)
3464 {
3465         int i;
3466
3467         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3468
3469         t4_teardown_adapter_queues(sc);
3470
3471         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3472                 taskqueue_free(sc->tq[i]);
3473                 sc->tq[i] = NULL;
3474         }
3475
3476         sc->flags &= ~FULL_INIT_DONE;
3477
3478         return (0);
3479 }
3480
3481 int
3482 port_full_init(struct port_info *pi)
3483 {
3484         struct adapter *sc = pi->adapter;
3485         struct ifnet *ifp = pi->ifp;
3486         uint16_t *rss;
3487         struct sge_rxq *rxq;
3488         int rc, i, j;
3489
3490         ASSERT_SYNCHRONIZED_OP(sc);
3491         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3492             ("%s: PORT_INIT_DONE already", __func__));
3493
3494         sysctl_ctx_init(&pi->ctx);
3495         pi->flags |= PORT_SYSCTL_CTX;
3496
3497         /*
3498          * Allocate tx/rx/fl queues for this port.
3499          */
3500         rc = t4_setup_port_queues(pi);
3501         if (rc != 0)
3502                 goto done;      /* error message displayed already */
3503
3504         /*
3505          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3506          */
3507         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3508         for (i = 0; i < pi->rss_size;) {
3509                 for_each_rxq(pi, j, rxq) {
3510                         rss[i++] = rxq->iq.abs_id;
3511                         if (i == pi->rss_size)
3512                                 break;
3513                 }
3514         }
3515
3516         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3517             pi->rss_size);
3518         if (rc != 0) {
3519                 if_printf(ifp, "rss_config failed: %d\n", rc);
3520                 goto done;
3521         }
3522
3523         pi->rss = rss;
3524         pi->flags |= PORT_INIT_DONE;
3525 done:
3526         if (rc != 0)
3527                 port_full_uninit(pi);
3528
3529         return (rc);
3530 }
3531
3532 /*
3533  * Idempotent.
3534  */
3535 int
3536 port_full_uninit(struct port_info *pi)
3537 {
3538         struct adapter *sc = pi->adapter;
3539         int i;
3540         struct sge_rxq *rxq;
3541         struct sge_txq *txq;
3542 #ifdef TCP_OFFLOAD
3543         struct sge_ofld_rxq *ofld_rxq;
3544         struct sge_wrq *ofld_txq;
3545 #endif
3546
3547         if (pi->flags & PORT_INIT_DONE) {
3548
3549                 /* Need to quiesce queues.  */
3550
3551                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3552
3553                 for_each_txq(pi, i, txq) {
3554                         quiesce_txq(sc, txq);
3555                 }
3556
3557 #ifdef TCP_OFFLOAD
3558                 for_each_ofld_txq(pi, i, ofld_txq) {
3559                         quiesce_wrq(sc, ofld_txq);
3560                 }
3561 #endif
3562
3563                 for_each_rxq(pi, i, rxq) {
3564                         quiesce_iq(sc, &rxq->iq);
3565                         quiesce_fl(sc, &rxq->fl);
3566                 }
3567
3568 #ifdef TCP_OFFLOAD
3569                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3570                         quiesce_iq(sc, &ofld_rxq->iq);
3571                         quiesce_fl(sc, &ofld_rxq->fl);
3572                 }
3573 #endif
3574                 free(pi->rss, M_CXGBE);
3575         }
3576
3577         t4_teardown_port_queues(pi);
3578         pi->flags &= ~PORT_INIT_DONE;
3579
3580         return (0);
3581 }
3582
3583 static void
3584 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3585 {
3586         struct sge_eq *eq = &txq->eq;
3587         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3588
3589         (void) sc;      /* unused */
3590
3591 #ifdef INVARIANTS
3592         TXQ_LOCK(txq);
3593         MPASS((eq->flags & EQ_ENABLED) == 0);
3594         TXQ_UNLOCK(txq);
3595 #endif
3596
3597         /* Wait for the mp_ring to empty. */
3598         while (!mp_ring_is_idle(txq->r)) {
3599                 mp_ring_check_drainage(txq->r, 0);
3600                 pause("rquiesce", 1);
3601         }
3602
3603         /* Then wait for the hardware to finish. */
3604         while (spg->cidx != htobe16(eq->pidx))
3605                 pause("equiesce", 1);
3606
3607         /* Finally, wait for the driver to reclaim all descriptors. */
3608         while (eq->cidx != eq->pidx)
3609                 pause("dquiesce", 1);
3610 }
3611
3612 static void
3613 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3614 {
3615
3616         /* XXXTX */
3617 }
3618
3619 static void
3620 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3621 {
3622         (void) sc;      /* unused */
3623
3624         /* Synchronize with the interrupt handler */
3625         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3626                 pause("iqfree", 1);
3627 }
3628
3629 static void
3630 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3631 {
3632         mtx_lock(&sc->sfl_lock);
3633         FL_LOCK(fl);
3634         fl->flags |= FL_DOOMED;
3635         FL_UNLOCK(fl);
3636         mtx_unlock(&sc->sfl_lock);
3637
3638         callout_drain(&sc->sfl_callout);
3639         KASSERT((fl->flags & FL_STARVING) == 0,
3640             ("%s: still starving", __func__));
3641 }
3642
3643 static int
3644 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3645     driver_intr_t *handler, void *arg, char *name)
3646 {
3647         int rc;
3648
3649         irq->rid = rid;
3650         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3651             RF_SHAREABLE | RF_ACTIVE);
3652         if (irq->res == NULL) {
3653                 device_printf(sc->dev,
3654                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3655                 return (ENOMEM);
3656         }
3657
3658         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3659             NULL, handler, arg, &irq->tag);
3660         if (rc != 0) {
3661                 device_printf(sc->dev,
3662                     "failed to setup interrupt for rid %d, name %s: %d\n",
3663                     rid, name, rc);
3664         } else if (name)
3665                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3666
3667         return (rc);
3668 }
3669
3670 static int
3671 t4_free_irq(struct adapter *sc, struct irq *irq)
3672 {
3673         if (irq->tag)
3674                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3675         if (irq->res)
3676                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3677
3678         bzero(irq, sizeof(*irq));
3679
3680         return (0);
3681 }
3682
3683 static void
3684 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3685     unsigned int end)
3686 {
3687         uint32_t *p = (uint32_t *)(buf + start);
3688
3689         for ( ; start <= end; start += sizeof(uint32_t))
3690                 *p++ = t4_read_reg(sc, start);
3691 }
3692
3693 static void
3694 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3695 {
3696         int i, n;
3697         const unsigned int *reg_ranges;
3698         static const unsigned int t4_reg_ranges[] = {
3699                 0x1008, 0x1108,
3700                 0x1180, 0x11b4,
3701                 0x11fc, 0x123c,
3702                 0x1300, 0x173c,
3703                 0x1800, 0x18fc,
3704                 0x3000, 0x30d8,
3705                 0x30e0, 0x5924,
3706                 0x5960, 0x59d4,
3707                 0x5a00, 0x5af8,
3708                 0x6000, 0x6098,
3709                 0x6100, 0x6150,
3710                 0x6200, 0x6208,
3711                 0x6240, 0x6248,
3712                 0x6280, 0x6338,
3713                 0x6370, 0x638c,
3714                 0x6400, 0x643c,
3715                 0x6500, 0x6524,
3716                 0x6a00, 0x6a38,
3717                 0x6a60, 0x6a78,
3718                 0x6b00, 0x6b84,
3719                 0x6bf0, 0x6c84,
3720                 0x6cf0, 0x6d84,
3721                 0x6df0, 0x6e84,
3722                 0x6ef0, 0x6f84,
3723                 0x6ff0, 0x7084,
3724                 0x70f0, 0x7184,
3725                 0x71f0, 0x7284,
3726                 0x72f0, 0x7384,
3727                 0x73f0, 0x7450,
3728                 0x7500, 0x7530,
3729                 0x7600, 0x761c,
3730                 0x7680, 0x76cc,
3731                 0x7700, 0x7798,
3732                 0x77c0, 0x77fc,
3733                 0x7900, 0x79fc,
3734                 0x7b00, 0x7c38,
3735                 0x7d00, 0x7efc,
3736                 0x8dc0, 0x8e1c,
3737                 0x8e30, 0x8e78,
3738                 0x8ea0, 0x8f6c,
3739                 0x8fc0, 0x9074,
3740                 0x90fc, 0x90fc,
3741                 0x9400, 0x9458,
3742                 0x9600, 0x96bc,
3743                 0x9800, 0x9808,
3744                 0x9820, 0x983c,
3745                 0x9850, 0x9864,
3746                 0x9c00, 0x9c6c,
3747                 0x9c80, 0x9cec,
3748                 0x9d00, 0x9d6c,
3749                 0x9d80, 0x9dec,
3750                 0x9e00, 0x9e6c,
3751                 0x9e80, 0x9eec,
3752                 0x9f00, 0x9f6c,
3753                 0x9f80, 0x9fec,
3754                 0xd004, 0xd03c,
3755                 0xdfc0, 0xdfe0,
3756                 0xe000, 0xea7c,
3757                 0xf000, 0x11110,
3758                 0x11118, 0x11190,
3759                 0x19040, 0x1906c,
3760                 0x19078, 0x19080,
3761                 0x1908c, 0x19124,
3762                 0x19150, 0x191b0,
3763                 0x191d0, 0x191e8,
3764                 0x19238, 0x1924c,
3765                 0x193f8, 0x19474,
3766                 0x19490, 0x194f8,
3767                 0x19800, 0x19f30,
3768                 0x1a000, 0x1a06c,
3769                 0x1a0b0, 0x1a120,
3770                 0x1a128, 0x1a138,
3771                 0x1a190, 0x1a1c4,
3772                 0x1a1fc, 0x1a1fc,
3773                 0x1e040, 0x1e04c,
3774                 0x1e284, 0x1e28c,
3775                 0x1e2c0, 0x1e2c0,
3776                 0x1e2e0, 0x1e2e0,
3777                 0x1e300, 0x1e384,
3778                 0x1e3c0, 0x1e3c8,
3779                 0x1e440, 0x1e44c,
3780                 0x1e684, 0x1e68c,
3781                 0x1e6c0, 0x1e6c0,
3782                 0x1e6e0, 0x1e6e0,
3783                 0x1e700, 0x1e784,
3784                 0x1e7c0, 0x1e7c8,
3785                 0x1e840, 0x1e84c,
3786                 0x1ea84, 0x1ea8c,
3787                 0x1eac0, 0x1eac0,
3788                 0x1eae0, 0x1eae0,
3789                 0x1eb00, 0x1eb84,
3790                 0x1ebc0, 0x1ebc8,
3791                 0x1ec40, 0x1ec4c,
3792                 0x1ee84, 0x1ee8c,
3793                 0x1eec0, 0x1eec0,
3794                 0x1eee0, 0x1eee0,
3795                 0x1ef00, 0x1ef84,
3796                 0x1efc0, 0x1efc8,
3797                 0x1f040, 0x1f04c,
3798                 0x1f284, 0x1f28c,
3799                 0x1f2c0, 0x1f2c0,
3800                 0x1f2e0, 0x1f2e0,
3801                 0x1f300, 0x1f384,
3802                 0x1f3c0, 0x1f3c8,
3803                 0x1f440, 0x1f44c,
3804                 0x1f684, 0x1f68c,
3805                 0x1f6c0, 0x1f6c0,
3806                 0x1f6e0, 0x1f6e0,
3807                 0x1f700, 0x1f784,
3808                 0x1f7c0, 0x1f7c8,
3809                 0x1f840, 0x1f84c,
3810                 0x1fa84, 0x1fa8c,
3811                 0x1fac0, 0x1fac0,
3812                 0x1fae0, 0x1fae0,
3813                 0x1fb00, 0x1fb84,
3814                 0x1fbc0, 0x1fbc8,
3815                 0x1fc40, 0x1fc4c,
3816                 0x1fe84, 0x1fe8c,
3817                 0x1fec0, 0x1fec0,
3818                 0x1fee0, 0x1fee0,
3819                 0x1ff00, 0x1ff84,
3820                 0x1ffc0, 0x1ffc8,
3821                 0x20000, 0x2002c,
3822                 0x20100, 0x2013c,
3823                 0x20190, 0x201c8,
3824                 0x20200, 0x20318,
3825                 0x20400, 0x20528,
3826                 0x20540, 0x20614,
3827                 0x21000, 0x21040,
3828                 0x2104c, 0x21060,
3829                 0x210c0, 0x210ec,
3830                 0x21200, 0x21268,
3831                 0x21270, 0x21284,
3832                 0x212fc, 0x21388,
3833                 0x21400, 0x21404,
3834                 0x21500, 0x21518,
3835                 0x2152c, 0x2153c,
3836                 0x21550, 0x21554,
3837                 0x21600, 0x21600,
3838                 0x21608, 0x21628,
3839                 0x21630, 0x2163c,
3840                 0x21700, 0x2171c,
3841                 0x21780, 0x2178c,
3842                 0x21800, 0x21c38,
3843                 0x21c80, 0x21d7c,
3844                 0x21e00, 0x21e04,
3845                 0x22000, 0x2202c,
3846                 0x22100, 0x2213c,
3847                 0x22190, 0x221c8,
3848                 0x22200, 0x22318,
3849                 0x22400, 0x22528,
3850                 0x22540, 0x22614,
3851                 0x23000, 0x23040,
3852                 0x2304c, 0x23060,
3853                 0x230c0, 0x230ec,
3854                 0x23200, 0x23268,
3855                 0x23270, 0x23284,
3856                 0x232fc, 0x23388,
3857                 0x23400, 0x23404,
3858                 0x23500, 0x23518,
3859                 0x2352c, 0x2353c,
3860                 0x23550, 0x23554,
3861                 0x23600, 0x23600,
3862                 0x23608, 0x23628,
3863                 0x23630, 0x2363c,
3864                 0x23700, 0x2371c,
3865                 0x23780, 0x2378c,
3866                 0x23800, 0x23c38,
3867                 0x23c80, 0x23d7c,
3868                 0x23e00, 0x23e04,
3869                 0x24000, 0x2402c,
3870                 0x24100, 0x2413c,
3871                 0x24190, 0x241c8,
3872                 0x24200, 0x24318,
3873                 0x24400, 0x24528,
3874                 0x24540, 0x24614,
3875                 0x25000, 0x25040,
3876                 0x2504c, 0x25060,
3877                 0x250c0, 0x250ec,
3878                 0x25200, 0x25268,
3879                 0x25270, 0x25284,
3880                 0x252fc, 0x25388,
3881                 0x25400, 0x25404,
3882                 0x25500, 0x25518,
3883                 0x2552c, 0x2553c,
3884                 0x25550, 0x25554,
3885                 0x25600, 0x25600,
3886                 0x25608, 0x25628,
3887                 0x25630, 0x2563c,
3888                 0x25700, 0x2571c,
3889                 0x25780, 0x2578c,
3890                 0x25800, 0x25c38,
3891                 0x25c80, 0x25d7c,
3892                 0x25e00, 0x25e04,
3893                 0x26000, 0x2602c,
3894                 0x26100, 0x2613c,
3895                 0x26190, 0x261c8,
3896                 0x26200, 0x26318,
3897                 0x26400, 0x26528,
3898                 0x26540, 0x26614,
3899                 0x27000, 0x27040,
3900                 0x2704c, 0x27060,
3901                 0x270c0, 0x270ec,
3902                 0x27200, 0x27268,
3903                 0x27270, 0x27284,
3904                 0x272fc, 0x27388,
3905                 0x27400, 0x27404,
3906                 0x27500, 0x27518,
3907                 0x2752c, 0x2753c,
3908                 0x27550, 0x27554,
3909                 0x27600, 0x27600,
3910                 0x27608, 0x27628,
3911                 0x27630, 0x2763c,
3912                 0x27700, 0x2771c,
3913                 0x27780, 0x2778c,
3914                 0x27800, 0x27c38,
3915                 0x27c80, 0x27d7c,
3916                 0x27e00, 0x27e04
3917         };
3918         static const unsigned int t5_reg_ranges[] = {
3919                 0x1008, 0x1148,
3920                 0x1180, 0x11b4,
3921                 0x11fc, 0x123c,
3922                 0x1280, 0x173c,
3923                 0x1800, 0x18fc,
3924                 0x3000, 0x3028,
3925                 0x3060, 0x30d8,
3926                 0x30e0, 0x30fc,
3927                 0x3140, 0x357c,
3928                 0x35a8, 0x35cc,
3929                 0x35ec, 0x35ec,
3930                 0x3600, 0x5624,
3931                 0x56cc, 0x575c,
3932                 0x580c, 0x5814,
3933                 0x5890, 0x58bc,
3934                 0x5940, 0x59dc,
3935                 0x59fc, 0x5a18,
3936                 0x5a60, 0x5a9c,
3937                 0x5b94, 0x5bfc,
3938                 0x6000, 0x6040,
3939                 0x6058, 0x614c,
3940                 0x7700, 0x7798,
3941                 0x77c0, 0x78fc,
3942                 0x7b00, 0x7c54,
3943                 0x7d00, 0x7efc,
3944                 0x8dc0, 0x8de0,
3945                 0x8df8, 0x8e84,
3946                 0x8ea0, 0x8f84,
3947                 0x8fc0, 0x90f8,
3948                 0x9400, 0x9470,
3949                 0x9600, 0x96f4,
3950                 0x9800, 0x9808,
3951                 0x9820, 0x983c,
3952                 0x9850, 0x9864,
3953                 0x9c00, 0x9c6c,
3954                 0x9c80, 0x9cec,
3955                 0x9d00, 0x9d6c,
3956                 0x9d80, 0x9dec,
3957                 0x9e00, 0x9e6c,
3958                 0x9e80, 0x9eec,
3959                 0x9f00, 0x9f6c,
3960                 0x9f80, 0xa020,
3961                 0xd004, 0xd03c,
3962                 0xdfc0, 0xdfe0,
3963                 0xe000, 0x11088,
3964                 0x1109c, 0x11110,
3965                 0x11118, 0x1117c,
3966                 0x11190, 0x11204,
3967                 0x19040, 0x1906c,
3968                 0x19078, 0x19080,
3969                 0x1908c, 0x19124,
3970                 0x19150, 0x191b0,
3971                 0x191d0, 0x191e8,
3972                 0x19238, 0x19290,
3973                 0x193f8, 0x19474,
3974                 0x19490, 0x194cc,
3975                 0x194f0, 0x194f8,
3976                 0x19c00, 0x19c60,
3977                 0x19c94, 0x19e10,
3978                 0x19e50, 0x19f34,
3979                 0x19f40, 0x19f50,
3980                 0x19f90, 0x19fe4,
3981                 0x1a000, 0x1a06c,
3982                 0x1a0b0, 0x1a120,
3983                 0x1a128, 0x1a138,
3984                 0x1a190, 0x1a1c4,
3985                 0x1a1fc, 0x1a1fc,
3986                 0x1e008, 0x1e00c,
3987                 0x1e040, 0x1e04c,
3988                 0x1e284, 0x1e290,
3989                 0x1e2c0, 0x1e2c0,
3990                 0x1e2e0, 0x1e2e0,
3991                 0x1e300, 0x1e384,
3992                 0x1e3c0, 0x1e3c8,
3993                 0x1e408, 0x1e40c,
3994                 0x1e440, 0x1e44c,
3995                 0x1e684, 0x1e690,
3996                 0x1e6c0, 0x1e6c0,
3997                 0x1e6e0, 0x1e6e0,
3998                 0x1e700, 0x1e784,
3999                 0x1e7c0, 0x1e7c8,
4000                 0x1e808, 0x1e80c,
4001                 0x1e840, 0x1e84c,
4002                 0x1ea84, 0x1ea90,
4003                 0x1eac0, 0x1eac0,
4004                 0x1eae0, 0x1eae0,
4005                 0x1eb00, 0x1eb84,
4006                 0x1ebc0, 0x1ebc8,
4007                 0x1ec08, 0x1ec0c,
4008                 0x1ec40, 0x1ec4c,
4009                 0x1ee84, 0x1ee90,
4010                 0x1eec0, 0x1eec0,
4011                 0x1eee0, 0x1eee0,
4012                 0x1ef00, 0x1ef84,
4013                 0x1efc0, 0x1efc8,
4014                 0x1f008, 0x1f00c,
4015                 0x1f040, 0x1f04c,
4016                 0x1f284, 0x1f290,
4017                 0x1f2c0, 0x1f2c0,
4018                 0x1f2e0, 0x1f2e0,
4019                 0x1f300, 0x1f384,
4020                 0x1f3c0, 0x1f3c8,
4021                 0x1f408, 0x1f40c,
4022                 0x1f440, 0x1f44c,
4023                 0x1f684, 0x1f690,
4024                 0x1f6c0, 0x1f6c0,
4025                 0x1f6e0, 0x1f6e0,
4026                 0x1f700, 0x1f784,
4027                 0x1f7c0, 0x1f7c8,
4028                 0x1f808, 0x1f80c,
4029                 0x1f840, 0x1f84c,
4030                 0x1fa84, 0x1fa90,
4031                 0x1fac0, 0x1fac0,
4032                 0x1fae0, 0x1fae0,
4033                 0x1fb00, 0x1fb84,
4034                 0x1fbc0, 0x1fbc8,
4035                 0x1fc08, 0x1fc0c,
4036                 0x1fc40, 0x1fc4c,
4037                 0x1fe84, 0x1fe90,
4038                 0x1fec0, 0x1fec0,
4039                 0x1fee0, 0x1fee0,
4040                 0x1ff00, 0x1ff84,
4041                 0x1ffc0, 0x1ffc8,
4042                 0x30000, 0x30030,
4043                 0x30100, 0x30144,
4044                 0x30190, 0x301d0,
4045                 0x30200, 0x30318,
4046                 0x30400, 0x3052c,
4047                 0x30540, 0x3061c,
4048                 0x30800, 0x30834,
4049                 0x308c0, 0x30908,
4050                 0x30910, 0x309ac,
4051                 0x30a00, 0x30a2c,
4052                 0x30a44, 0x30a50,
4053                 0x30a74, 0x30c24,
4054                 0x30d00, 0x30d00,
4055                 0x30d08, 0x30d14,
4056                 0x30d1c, 0x30d20,
4057                 0x30d3c, 0x30d50,
4058                 0x31200, 0x3120c,
4059                 0x31220, 0x31220,
4060                 0x31240, 0x31240,
4061                 0x31600, 0x3160c,
4062                 0x31a00, 0x31a1c,
4063                 0x31e00, 0x31e20,
4064                 0x31e38, 0x31e3c,
4065                 0x31e80, 0x31e80,
4066                 0x31e88, 0x31ea8,
4067                 0x31eb0, 0x31eb4,
4068                 0x31ec8, 0x31ed4,
4069                 0x31fb8, 0x32004,
4070                 0x32200, 0x32200,
4071                 0x32208, 0x32240,
4072                 0x32248, 0x32280,
4073                 0x32288, 0x322c0,
4074                 0x322c8, 0x322fc,
4075                 0x32600, 0x32630,
4076                 0x32a00, 0x32abc,
4077                 0x32b00, 0x32b70,
4078                 0x33000, 0x33048,
4079                 0x33060, 0x3309c,
4080                 0x330f0, 0x33148,
4081                 0x33160, 0x3319c,
4082                 0x331f0, 0x332e4,
4083                 0x332f8, 0x333e4,
4084                 0x333f8, 0x33448,
4085                 0x33460, 0x3349c,
4086                 0x334f0, 0x33548,
4087                 0x33560, 0x3359c,
4088                 0x335f0, 0x336e4,
4089                 0x336f8, 0x337e4,
4090                 0x337f8, 0x337fc,
4091                 0x33814, 0x33814,
4092                 0x3382c, 0x3382c,
4093                 0x33880, 0x3388c,
4094                 0x338e8, 0x338ec,
4095                 0x33900, 0x33948,
4096                 0x33960, 0x3399c,
4097                 0x339f0, 0x33ae4,
4098                 0x33af8, 0x33b10,
4099                 0x33b28, 0x33b28,
4100                 0x33b3c, 0x33b50,
4101                 0x33bf0, 0x33c10,
4102                 0x33c28, 0x33c28,
4103                 0x33c3c, 0x33c50,
4104                 0x33cf0, 0x33cfc,
4105                 0x34000, 0x34030,
4106                 0x34100, 0x34144,
4107                 0x34190, 0x341d0,
4108                 0x34200, 0x34318,
4109                 0x34400, 0x3452c,
4110                 0x34540, 0x3461c,
4111                 0x34800, 0x34834,
4112                 0x348c0, 0x34908,
4113                 0x34910, 0x349ac,
4114                 0x34a00, 0x34a2c,
4115                 0x34a44, 0x34a50,
4116                 0x34a74, 0x34c24,
4117                 0x34d00, 0x34d00,
4118                 0x34d08, 0x34d14,
4119                 0x34d1c, 0x34d20,
4120                 0x34d3c, 0x34d50,
4121                 0x35200, 0x3520c,
4122                 0x35220, 0x35220,
4123                 0x35240, 0x35240,
4124                 0x35600, 0x3560c,
4125                 0x35a00, 0x35a1c,
4126                 0x35e00, 0x35e20,
4127                 0x35e38, 0x35e3c,
4128                 0x35e80, 0x35e80,
4129                 0x35e88, 0x35ea8,
4130                 0x35eb0, 0x35eb4,
4131                 0x35ec8, 0x35ed4,
4132                 0x35fb8, 0x36004,
4133                 0x36200, 0x36200,
4134                 0x36208, 0x36240,
4135                 0x36248, 0x36280,
4136                 0x36288, 0x362c0,
4137                 0x362c8, 0x362fc,
4138                 0x36600, 0x36630,
4139                 0x36a00, 0x36abc,
4140                 0x36b00, 0x36b70,
4141                 0x37000, 0x37048,
4142                 0x37060, 0x3709c,
4143                 0x370f0, 0x37148,
4144                 0x37160, 0x3719c,
4145                 0x371f0, 0x372e4,
4146                 0x372f8, 0x373e4,
4147                 0x373f8, 0x37448,
4148                 0x37460, 0x3749c,
4149                 0x374f0, 0x37548,
4150                 0x37560, 0x3759c,
4151                 0x375f0, 0x376e4,
4152                 0x376f8, 0x377e4,
4153                 0x377f8, 0x377fc,
4154                 0x37814, 0x37814,
4155                 0x3782c, 0x3782c,
4156                 0x37880, 0x3788c,
4157                 0x378e8, 0x378ec,
4158                 0x37900, 0x37948,
4159                 0x37960, 0x3799c,
4160                 0x379f0, 0x37ae4,
4161                 0x37af8, 0x37b10,
4162                 0x37b28, 0x37b28,
4163                 0x37b3c, 0x37b50,
4164                 0x37bf0, 0x37c10,
4165                 0x37c28, 0x37c28,
4166                 0x37c3c, 0x37c50,
4167                 0x37cf0, 0x37cfc,
4168                 0x38000, 0x38030,
4169                 0x38100, 0x38144,
4170                 0x38190, 0x381d0,
4171                 0x38200, 0x38318,
4172                 0x38400, 0x3852c,
4173                 0x38540, 0x3861c,
4174                 0x38800, 0x38834,
4175                 0x388c0, 0x38908,
4176                 0x38910, 0x389ac,
4177                 0x38a00, 0x38a2c,
4178                 0x38a44, 0x38a50,
4179                 0x38a74, 0x38c24,
4180                 0x38d00, 0x38d00,
4181                 0x38d08, 0x38d14,
4182                 0x38d1c, 0x38d20,
4183                 0x38d3c, 0x38d50,
4184                 0x39200, 0x3920c,
4185                 0x39220, 0x39220,
4186                 0x39240, 0x39240,
4187                 0x39600, 0x3960c,
4188                 0x39a00, 0x39a1c,
4189                 0x39e00, 0x39e20,
4190                 0x39e38, 0x39e3c,
4191                 0x39e80, 0x39e80,
4192                 0x39e88, 0x39ea8,
4193                 0x39eb0, 0x39eb4,
4194                 0x39ec8, 0x39ed4,
4195                 0x39fb8, 0x3a004,
4196                 0x3a200, 0x3a200,
4197                 0x3a208, 0x3a240,
4198                 0x3a248, 0x3a280,
4199                 0x3a288, 0x3a2c0,
4200                 0x3a2c8, 0x3a2fc,
4201                 0x3a600, 0x3a630,
4202                 0x3aa00, 0x3aabc,
4203                 0x3ab00, 0x3ab70,
4204                 0x3b000, 0x3b048,
4205                 0x3b060, 0x3b09c,
4206                 0x3b0f0, 0x3b148,
4207                 0x3b160, 0x3b19c,
4208                 0x3b1f0, 0x3b2e4,
4209                 0x3b2f8, 0x3b3e4,
4210                 0x3b3f8, 0x3b448,
4211                 0x3b460, 0x3b49c,
4212                 0x3b4f0, 0x3b548,
4213                 0x3b560, 0x3b59c,
4214                 0x3b5f0, 0x3b6e4,
4215                 0x3b6f8, 0x3b7e4,
4216                 0x3b7f8, 0x3b7fc,
4217                 0x3b814, 0x3b814,
4218                 0x3b82c, 0x3b82c,
4219                 0x3b880, 0x3b88c,
4220                 0x3b8e8, 0x3b8ec,
4221                 0x3b900, 0x3b948,
4222                 0x3b960, 0x3b99c,
4223                 0x3b9f0, 0x3bae4,
4224                 0x3baf8, 0x3bb10,
4225                 0x3bb28, 0x3bb28,
4226                 0x3bb3c, 0x3bb50,
4227                 0x3bbf0, 0x3bc10,
4228                 0x3bc28, 0x3bc28,
4229                 0x3bc3c, 0x3bc50,
4230                 0x3bcf0, 0x3bcfc,
4231                 0x3c000, 0x3c030,
4232                 0x3c100, 0x3c144,
4233                 0x3c190, 0x3c1d0,
4234                 0x3c200, 0x3c318,
4235                 0x3c400, 0x3c52c,
4236                 0x3c540, 0x3c61c,
4237                 0x3c800, 0x3c834,
4238                 0x3c8c0, 0x3c908,
4239                 0x3c910, 0x3c9ac,
4240                 0x3ca00, 0x3ca2c,
4241                 0x3ca44, 0x3ca50,
4242                 0x3ca74, 0x3cc24,
4243                 0x3cd00, 0x3cd00,
4244                 0x3cd08, 0x3cd14,
4245                 0x3cd1c, 0x3cd20,
4246                 0x3cd3c, 0x3cd50,
4247                 0x3d200, 0x3d20c,
4248                 0x3d220, 0x3d220,
4249                 0x3d240, 0x3d240,
4250                 0x3d600, 0x3d60c,
4251                 0x3da00, 0x3da1c,
4252                 0x3de00, 0x3de20,
4253                 0x3de38, 0x3de3c,
4254                 0x3de80, 0x3de80,
4255                 0x3de88, 0x3dea8,
4256                 0x3deb0, 0x3deb4,
4257                 0x3dec8, 0x3ded4,
4258                 0x3dfb8, 0x3e004,
4259                 0x3e200, 0x3e200,
4260                 0x3e208, 0x3e240,
4261                 0x3e248, 0x3e280,
4262                 0x3e288, 0x3e2c0,
4263                 0x3e2c8, 0x3e2fc,
4264                 0x3e600, 0x3e630,
4265                 0x3ea00, 0x3eabc,
4266                 0x3eb00, 0x3eb70,
4267                 0x3f000, 0x3f048,
4268                 0x3f060, 0x3f09c,
4269                 0x3f0f0, 0x3f148,
4270                 0x3f160, 0x3f19c,
4271                 0x3f1f0, 0x3f2e4,
4272                 0x3f2f8, 0x3f3e4,
4273                 0x3f3f8, 0x3f448,
4274                 0x3f460, 0x3f49c,
4275                 0x3f4f0, 0x3f548,
4276                 0x3f560, 0x3f59c,
4277                 0x3f5f0, 0x3f6e4,
4278                 0x3f6f8, 0x3f7e4,
4279                 0x3f7f8, 0x3f7fc,
4280                 0x3f814, 0x3f814,
4281                 0x3f82c, 0x3f82c,
4282                 0x3f880, 0x3f88c,
4283                 0x3f8e8, 0x3f8ec,
4284                 0x3f900, 0x3f948,
4285                 0x3f960, 0x3f99c,
4286                 0x3f9f0, 0x3fae4,
4287                 0x3faf8, 0x3fb10,
4288                 0x3fb28, 0x3fb28,
4289                 0x3fb3c, 0x3fb50,
4290                 0x3fbf0, 0x3fc10,
4291                 0x3fc28, 0x3fc28,
4292                 0x3fc3c, 0x3fc50,
4293                 0x3fcf0, 0x3fcfc,
4294                 0x40000, 0x4000c,
4295                 0x40040, 0x40068,
4296                 0x4007c, 0x40144,
4297                 0x40180, 0x4018c,
4298                 0x40200, 0x40298,
4299                 0x402ac, 0x4033c,
4300                 0x403f8, 0x403fc,
4301                 0x41304, 0x413c4,
4302                 0x41400, 0x4141c,
4303                 0x41480, 0x414d0,
4304                 0x44000, 0x44078,
4305                 0x440c0, 0x44278,
4306                 0x442c0, 0x44478,
4307                 0x444c0, 0x44678,
4308                 0x446c0, 0x44878,
4309                 0x448c0, 0x449fc,
4310                 0x45000, 0x45068,
4311                 0x45080, 0x45084,
4312                 0x450a0, 0x450b0,
4313                 0x45200, 0x45268,
4314                 0x45280, 0x45284,
4315                 0x452a0, 0x452b0,
4316                 0x460c0, 0x460e4,
4317                 0x47000, 0x4708c,
4318                 0x47200, 0x47250,
4319                 0x47400, 0x47420,
4320                 0x47600, 0x47618,
4321                 0x47800, 0x47814,
4322                 0x48000, 0x4800c,
4323                 0x48040, 0x48068,
4324                 0x4807c, 0x48144,
4325                 0x48180, 0x4818c,
4326                 0x48200, 0x48298,
4327                 0x482ac, 0x4833c,
4328                 0x483f8, 0x483fc,
4329                 0x49304, 0x493c4,
4330                 0x49400, 0x4941c,
4331                 0x49480, 0x494d0,
4332                 0x4c000, 0x4c078,
4333                 0x4c0c0, 0x4c278,
4334                 0x4c2c0, 0x4c478,
4335                 0x4c4c0, 0x4c678,
4336                 0x4c6c0, 0x4c878,
4337                 0x4c8c0, 0x4c9fc,
4338                 0x4d000, 0x4d068,
4339                 0x4d080, 0x4d084,
4340                 0x4d0a0, 0x4d0b0,
4341                 0x4d200, 0x4d268,
4342                 0x4d280, 0x4d284,
4343                 0x4d2a0, 0x4d2b0,
4344                 0x4e0c0, 0x4e0e4,
4345                 0x4f000, 0x4f08c,
4346                 0x4f200, 0x4f250,
4347                 0x4f400, 0x4f420,
4348                 0x4f600, 0x4f618,
4349                 0x4f800, 0x4f814,
4350                 0x50000, 0x500cc,
4351                 0x50400, 0x50400,
4352                 0x50800, 0x508cc,
4353                 0x50c00, 0x50c00,
4354                 0x51000, 0x5101c,
4355                 0x51300, 0x51308,
4356         };
4357
4358         if (is_t4(sc)) {
4359                 reg_ranges = &t4_reg_ranges[0];
4360                 n = nitems(t4_reg_ranges);
4361         } else {
4362                 reg_ranges = &t5_reg_ranges[0];
4363                 n = nitems(t5_reg_ranges);
4364         }
4365
4366         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4367         for (i = 0; i < n; i += 2)
4368                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4369 }
4370
4371 static void
4372 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4373 {
4374         int i;
4375         u_int v, tnl_cong_drops;
4376         struct timeval tv;
4377         const struct timeval interval = {0, 250000};    /* 250ms */
4378
4379         getmicrotime(&tv);
4380         timevalsub(&tv, &interval);
4381         if (timevalcmp(&tv, &pi->last_refreshed, <))
4382                 return;
4383
4384         tnl_cong_drops = 0;
4385         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4386         for (i = 0; i < NCHAN; i++) {
4387                 if (pi->rx_chan_map & (1 << i)) {
4388                         mtx_lock(&sc->regwin_lock);
4389                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4390                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4391                         mtx_unlock(&sc->regwin_lock);
4392                         tnl_cong_drops += v;
4393                 }
4394         }
4395         pi->tnl_cong_drops = tnl_cong_drops;
4396         getmicrotime(&pi->last_refreshed);
4397 }
4398
4399 static void
4400 cxgbe_tick(void *arg)
4401 {
4402         struct port_info *pi = arg;
4403         struct adapter *sc = pi->adapter;
4404         struct ifnet *ifp = pi->ifp;
4405
4406         PORT_LOCK(pi);
4407         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4408                 PORT_UNLOCK(pi);
4409                 return; /* without scheduling another callout */
4410         }
4411
4412         cxgbe_refresh_stats(sc, pi);
4413
4414         callout_schedule(&pi->tick, hz);
4415         PORT_UNLOCK(pi);
4416 }
4417
4418 static void
4419 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4420 {
4421         struct ifnet *vlan;
4422
4423         if (arg != ifp || ifp->if_type != IFT_ETHER)
4424                 return;
4425
4426         vlan = VLAN_DEVAT(ifp, vid);
4427         VLAN_SETCOOKIE(vlan, ifp);
4428 }
4429
4430 static int
4431 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4432 {
4433
4434 #ifdef INVARIANTS
4435         panic("%s: opcode 0x%02x on iq %p with payload %p",
4436             __func__, rss->opcode, iq, m);
4437 #else
4438         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4439             __func__, rss->opcode, iq, m);
4440         m_freem(m);
4441 #endif
4442         return (EDOOFUS);
4443 }
4444
4445 int
4446 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4447 {
4448         uintptr_t *loc, new;
4449
4450         if (opcode >= nitems(sc->cpl_handler))
4451                 return (EINVAL);
4452
4453         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4454         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4455         atomic_store_rel_ptr(loc, new);
4456
4457         return (0);
4458 }
4459
4460 static int
4461 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4462 {
4463
4464 #ifdef INVARIANTS
4465         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4466 #else
4467         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4468             __func__, iq, ctrl);
4469 #endif
4470         return (EDOOFUS);
4471 }
4472
4473 int
4474 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4475 {
4476         uintptr_t *loc, new;
4477
4478         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4479         loc = (uintptr_t *) &sc->an_handler;
4480         atomic_store_rel_ptr(loc, new);
4481
4482         return (0);
4483 }
4484
4485 static int
4486 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4487 {
4488         const struct cpl_fw6_msg *cpl =
4489             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4490
4491 #ifdef INVARIANTS
4492         panic("%s: fw_msg type %d", __func__, cpl->type);
4493 #else
4494         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4495 #endif
4496         return (EDOOFUS);
4497 }
4498
4499 int
4500 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4501 {
4502         uintptr_t *loc, new;
4503
4504         if (type >= nitems(sc->fw_msg_handler))
4505                 return (EINVAL);
4506
4507         /*
4508          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4509          * handler dispatch table.  Reject any attempt to install a handler for
4510          * this subtype.
4511          */
4512         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4513                 return (EINVAL);
4514
4515         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4516         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4517         atomic_store_rel_ptr(loc, new);
4518
4519         return (0);
4520 }
4521
4522 static int
4523 t4_sysctls(struct adapter *sc)
4524 {
4525         struct sysctl_ctx_list *ctx;
4526         struct sysctl_oid *oid;
4527         struct sysctl_oid_list *children, *c0;
4528         static char *caps[] = {
4529                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4530                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4531                     "\6HASHFILTER\7ETHOFLD",
4532                 "\20\1TOE",                             /* caps[2] toecaps */
4533                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4534                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4535                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4536                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4537                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4538                     "\4PO_INITIAOR\5PO_TARGET"
4539         };
4540         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4541
4542         ctx = device_get_sysctl_ctx(sc->dev);
4543
4544         /*
4545          * dev.t4nex.X.
4546          */
4547         oid = device_get_sysctl_tree(sc->dev);
4548         c0 = children = SYSCTL_CHILDREN(oid);
4549
4550         sc->sc_do_rxcopy = 1;
4551         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4552             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4553
4554         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4555             sc->params.nports, "# of ports");
4556
4557         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4558             NULL, chip_rev(sc), "chip hardware revision");
4559
4560         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4561             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4562
4563         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4564             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4565
4566         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4567             sc->cfcsum, "config file checksum");
4568
4569         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4570             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4571             sysctl_bitfield, "A", "available doorbells");
4572
4573         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4574             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4575             sysctl_bitfield, "A", "available link capabilities");
4576
4577         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4578             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4579             sysctl_bitfield, "A", "available NIC capabilities");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4582             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4583             sysctl_bitfield, "A", "available TCP offload capabilities");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4586             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4587             sysctl_bitfield, "A", "available RDMA capabilities");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4590             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4591             sysctl_bitfield, "A", "available iSCSI capabilities");
4592
4593         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4594             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4595             sysctl_bitfield, "A", "available FCoE capabilities");
4596
4597         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4598             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4599
4600         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4601             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4602             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4603             "interrupt holdoff timer values (us)");
4604
4605         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4606             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4607             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4608             "interrupt holdoff packet counter values");
4609
4610         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4611             NULL, sc->tids.nftids, "number of filters");
4612
4613         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4614             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4615             "chip temperature (in Celsius)");
4616
4617         t4_sge_sysctls(sc, ctx, children);
4618
4619         sc->lro_timeout = 100;
4620         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4621             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4622
4623 #ifdef SBUF_DRAIN
4624         /*
4625          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4626          */
4627         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4628             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4629             "logs and miscellaneous information");
4630         children = SYSCTL_CHILDREN(oid);
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634             sysctl_cctrl, "A", "congestion control");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4642             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4646             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4650             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4654             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4655
4656         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4657             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4658             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4659
4660         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4661             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4662             sysctl_cim_la, "A", "CIM logic analyzer");
4663
4664         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4665             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4666             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4667
4668         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4669             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4670             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4671
4672         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4673             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4674             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4675
4676         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4677             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4678             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4679
4680         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4681             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4682             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4683
4684         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4685             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4686             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4690             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4691
4692         if (is_t5(sc)) {
4693                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4694                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4695                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4696
4697                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4698                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4699                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4700         }
4701
4702         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4703             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4704             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4705
4706         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4707             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4708             sysctl_cim_qcfg, "A", "CIM queue configuration");
4709
4710         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4711             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4712             sysctl_cpl_stats, "A", "CPL statistics");
4713
4714         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4715             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4716             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4717
4718         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4719             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4720             sysctl_devlog, "A", "firmware's device log");
4721
4722         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4723             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4724             sysctl_fcoe_stats, "A", "FCoE statistics");
4725
4726         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4727             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4728             sysctl_hw_sched, "A", "hardware scheduler ");
4729
4730         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4731             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4732             sysctl_l2t, "A", "hardware L2 table");
4733
4734         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4735             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4736             sysctl_lb_stats, "A", "loopback statistics");
4737
4738         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4739             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4740             sysctl_meminfo, "A", "memory regions");
4741
4742         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4743             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4744             sysctl_mps_tcam, "A", "MPS TCAM entries");
4745
4746         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4747             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4748             sysctl_path_mtus, "A", "path MTUs");
4749
4750         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4751             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4752             sysctl_pm_stats, "A", "PM statistics");
4753
4754         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4755             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4756             sysctl_rdma_stats, "A", "RDMA statistics");
4757
4758         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4759             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4760             sysctl_tcp_stats, "A", "TCP statistics");
4761
4762         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4763             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4764             sysctl_tids, "A", "TID information");
4765
4766         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4767             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4768             sysctl_tp_err_stats, "A", "TP error statistics");
4769
4770         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4771             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4772             sysctl_tp_la, "A", "TP logic analyzer");
4773
4774         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4775             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4776             sysctl_tx_rate, "A", "Tx rate");
4777
4778         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4779             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4780             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4781
4782         if (is_t5(sc)) {
4783                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4784                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4785                     sysctl_wcwr_stats, "A", "write combined work requests");
4786         }
4787 #endif
4788
4789 #ifdef TCP_OFFLOAD
4790         if (is_offload(sc)) {
4791                 /*
4792                  * dev.t4nex.X.toe.
4793                  */
4794                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4795                     NULL, "TOE parameters");
4796                 children = SYSCTL_CHILDREN(oid);
4797
4798                 sc->tt.sndbuf = 256 * 1024;
4799                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4800                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4801
4802                 sc->tt.ddp = 0;
4803                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4804                     &sc->tt.ddp, 0, "DDP allowed");
4805
4806                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4807                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4808                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4809
4810                 sc->tt.ddp_thres =
4811                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4812                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4813                     &sc->tt.ddp_thres, 0, "DDP threshold");
4814
4815                 sc->tt.rx_coalesce = 1;
4816                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4817                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4818
4819                 sc->tt.tx_align = 1;
4820                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4821                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4822         }
4823 #endif
4824
4825
4826         return (0);
4827 }
4828
4829 static int
4830 cxgbe_sysctls(struct port_info *pi)
4831 {
4832         struct sysctl_ctx_list *ctx;
4833         struct sysctl_oid *oid;
4834         struct sysctl_oid_list *children;
4835         struct adapter *sc = pi->adapter;
4836
4837         ctx = device_get_sysctl_ctx(pi->dev);
4838
4839         /*
4840          * dev.cxgbe.X.
4841          */
4842         oid = device_get_sysctl_tree(pi->dev);
4843         children = SYSCTL_CHILDREN(oid);
4844
4845         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4846            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4847         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4848                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4849                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4850                     "PHY temperature (in Celsius)");
4851                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4852                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4853                     "PHY firmware version");
4854         }
4855         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4856             &pi->nrxq, 0, "# of rx queues");
4857         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4858             &pi->ntxq, 0, "# of tx queues");
4859         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4860             &pi->first_rxq, 0, "index of first rx queue");
4861         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4862             &pi->first_txq, 0, "index of first tx queue");
4863         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4864             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4865             "Reserve queue 0 for non-flowid packets");
4866
4867 #ifdef TCP_OFFLOAD
4868         if (is_offload(sc)) {
4869                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4870                     &pi->nofldrxq, 0,
4871                     "# of rx queues for offloaded TCP connections");
4872                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4873                     &pi->nofldtxq, 0,
4874                     "# of tx queues for offloaded TCP connections");
4875                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4876                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4877                     "index of first TOE rx queue");
4878                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4879                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4880                     "index of first TOE tx queue");
4881         }
4882 #endif
4883 #ifdef DEV_NETMAP
4884         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4885             &pi->nnmrxq, 0, "# of rx queues for netmap");
4886         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4887             &pi->nnmtxq, 0, "# of tx queues for netmap");
4888         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4889             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4890             "index of first netmap rx queue");
4891         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4892             CTLFLAG_RD, &pi->first_nm_txq, 0,
4893             "index of first netmap tx queue");
4894 #endif
4895
4896         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4897             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4898             "holdoff timer index");
4899         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4900             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4901             "holdoff packet counter index");
4902
4903         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4904             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4905             "rx queue size");
4906         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4907             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4908             "tx queue size");
4909
4910         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4911             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4912             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4913
4914         /*
4915          * dev.cxgbe.X.stats.
4916          */
4917         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4918             NULL, "port statistics");
4919         children = SYSCTL_CHILDREN(oid);
4920         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4921             &pi->tx_parse_error, 0,
4922             "# of tx packets with invalid length or # of segments");
4923
4924 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4925         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4926             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4927             sysctl_handle_t4_reg64, "QU", desc)
4928
4929         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4931         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4932             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4933         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4934             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4935         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4936             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4937         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4938             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4939         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4940             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4941         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4942             "# of tx frames in this range",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4944         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4945             "# of tx frames in this range",
4946             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4947         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4948             "# of tx frames in this range",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4951             "# of tx frames in this range",
4952             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4953         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4954             "# of tx frames in this range",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4956         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4957             "# of tx frames in this range",
4958             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4959         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4960             "# of tx frames in this range",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4962         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4964         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4966         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4968         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4969             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4970         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4971             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4972         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4973             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4974         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4975             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4976         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4977             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4978         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4979             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4980         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4981             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4982
4983         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4992             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4993         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4994             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4995         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4996             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4997         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4998             "# of frames received with bad FCS",
4999             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5000         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5001             "# of frames received with length error",
5002             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5003         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5004             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5005         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5006             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5007         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5008             "# of rx frames in this range",
5009             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5010         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5011             "# of rx frames in this range",
5012             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5013         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5014             "# of rx frames in this range",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5017             "# of rx frames in this range",
5018             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5019         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5020             "# of rx frames in this range",
5021             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5022         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5023             "# of rx frames in this range",
5024             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5025         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5026             "# of rx frames in this range",
5027             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5028         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5029             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5030         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5031             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5032         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5033             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5034         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5035             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5036         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5037             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5038         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5039             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5040         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5041             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5042         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5043             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5044         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5045             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5046
5047 #undef SYSCTL_ADD_T4_REG64
5048
5049 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5050         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5051             &pi->stats.name, desc)
5052
5053         /* We get these from port_stats and they may be stale by upto 1s */
5054         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5055             "# drops due to buffer-group 0 overflows");
5056         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5057             "# drops due to buffer-group 1 overflows");
5058         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5059             "# drops due to buffer-group 2 overflows");
5060         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5061             "# drops due to buffer-group 3 overflows");
5062         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5063             "# of buffer-group 0 truncated packets");
5064         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5065             "# of buffer-group 1 truncated packets");
5066         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5067             "# of buffer-group 2 truncated packets");
5068         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5069             "# of buffer-group 3 truncated packets");
5070
5071 #undef SYSCTL_ADD_T4_PORTSTAT
5072
5073         return (0);
5074 }
5075
5076 static int
5077 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5078 {
5079         int rc, *i;
5080         struct sbuf sb;
5081
5082         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5083         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
5084                 sbuf_printf(&sb, "%d ", *i);
5085         sbuf_trim(&sb);
5086         sbuf_finish(&sb);
5087         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5088         sbuf_delete(&sb);
5089         return (rc);
5090 }
5091
5092 static int
5093 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5094 {
5095         int rc;
5096         struct sbuf *sb;
5097
5098         rc = sysctl_wire_old_buffer(req, 0);
5099         if (rc != 0)
5100                 return(rc);
5101
5102         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5103         if (sb == NULL)
5104                 return (ENOMEM);
5105
5106         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5107         rc = sbuf_finish(sb);
5108         sbuf_delete(sb);
5109
5110         return (rc);
5111 }
5112
5113 static int
5114 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5115 {
5116         struct port_info *pi = arg1;
5117         int op = arg2;
5118         struct adapter *sc = pi->adapter;
5119         u_int v;
5120         int rc;
5121
5122         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5123         if (rc)
5124                 return (rc);
5125         /* XXX: magic numbers */
5126         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5127             &v);
5128         end_synchronized_op(sc, 0);
5129         if (rc)
5130                 return (rc);
5131         if (op == 0)
5132                 v /= 256;
5133
5134         rc = sysctl_handle_int(oidp, &v, 0, req);
5135         return (rc);
5136 }
5137
5138 static int
5139 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5140 {
5141         struct port_info *pi = arg1;
5142         int rc, val;
5143
5144         val = pi->rsrv_noflowq;
5145         rc = sysctl_handle_int(oidp, &val, 0, req);
5146         if (rc != 0 || req->newptr == NULL)
5147                 return (rc);
5148
5149         if ((val >= 1) && (pi->ntxq > 1))
5150                 pi->rsrv_noflowq = 1;
5151         else
5152                 pi->rsrv_noflowq = 0;
5153
5154         return (rc);
5155 }
5156
5157 static int
5158 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5159 {
5160         struct port_info *pi = arg1;
5161         struct adapter *sc = pi->adapter;
5162         int idx, rc, i;
5163         struct sge_rxq *rxq;
5164 #ifdef TCP_OFFLOAD
5165         struct sge_ofld_rxq *ofld_rxq;
5166 #endif
5167         uint8_t v;
5168
5169         idx = pi->tmr_idx;
5170
5171         rc = sysctl_handle_int(oidp, &idx, 0, req);
5172         if (rc != 0 || req->newptr == NULL)
5173                 return (rc);
5174
5175         if (idx < 0 || idx >= SGE_NTIMERS)
5176                 return (EINVAL);
5177
5178         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5179             "t4tmr");
5180         if (rc)
5181                 return (rc);
5182
5183         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5184         for_each_rxq(pi, i, rxq) {
5185 #ifdef atomic_store_rel_8
5186                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5187 #else
5188                 rxq->iq.intr_params = v;
5189 #endif
5190         }
5191 #ifdef TCP_OFFLOAD
5192         for_each_ofld_rxq(pi, i, ofld_rxq) {
5193 #ifdef atomic_store_rel_8
5194                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5195 #else
5196                 ofld_rxq->iq.intr_params = v;
5197 #endif
5198         }
5199 #endif
5200         pi->tmr_idx = idx;
5201
5202         end_synchronized_op(sc, LOCK_HELD);
5203         return (0);
5204 }
5205
5206 static int
5207 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5208 {
5209         struct port_info *pi = arg1;
5210         struct adapter *sc = pi->adapter;
5211         int idx, rc;
5212
5213         idx = pi->pktc_idx;
5214
5215         rc = sysctl_handle_int(oidp, &idx, 0, req);
5216         if (rc != 0 || req->newptr == NULL)
5217                 return (rc);
5218
5219         if (idx < -1 || idx >= SGE_NCOUNTERS)
5220                 return (EINVAL);
5221
5222         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5223             "t4pktc");
5224         if (rc)
5225                 return (rc);
5226
5227         if (pi->flags & PORT_INIT_DONE)
5228                 rc = EBUSY; /* cannot be changed once the queues are created */
5229         else
5230                 pi->pktc_idx = idx;
5231
5232         end_synchronized_op(sc, LOCK_HELD);
5233         return (rc);
5234 }
5235
5236 static int
5237 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5238 {
5239         struct port_info *pi = arg1;
5240         struct adapter *sc = pi->adapter;
5241         int qsize, rc;
5242
5243         qsize = pi->qsize_rxq;
5244
5245         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5246         if (rc != 0 || req->newptr == NULL)
5247                 return (rc);
5248
5249         if (qsize < 128 || (qsize & 7))
5250                 return (EINVAL);
5251
5252         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5253             "t4rxqs");
5254         if (rc)
5255                 return (rc);
5256
5257         if (pi->flags & PORT_INIT_DONE)
5258                 rc = EBUSY; /* cannot be changed once the queues are created */
5259         else
5260                 pi->qsize_rxq = qsize;
5261
5262         end_synchronized_op(sc, LOCK_HELD);
5263         return (rc);
5264 }
5265
5266 static int
5267 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5268 {
5269         struct port_info *pi = arg1;
5270         struct adapter *sc = pi->adapter;
5271         int qsize, rc;
5272
5273         qsize = pi->qsize_txq;
5274
5275         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5276         if (rc != 0 || req->newptr == NULL)
5277                 return (rc);
5278
5279         if (qsize < 128 || qsize > 65536)
5280                 return (EINVAL);
5281
5282         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5283             "t4txqs");
5284         if (rc)
5285                 return (rc);
5286
5287         if (pi->flags & PORT_INIT_DONE)
5288                 rc = EBUSY; /* cannot be changed once the queues are created */
5289         else
5290                 pi->qsize_txq = qsize;
5291
5292         end_synchronized_op(sc, LOCK_HELD);
5293         return (rc);
5294 }
5295
5296 static int
5297 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5298 {
5299         struct port_info *pi = arg1;
5300         struct adapter *sc = pi->adapter;
5301         struct link_config *lc = &pi->link_cfg;
5302         int rc;
5303
5304         if (req->newptr == NULL) {
5305                 struct sbuf *sb;
5306                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5307
5308                 rc = sysctl_wire_old_buffer(req, 0);
5309                 if (rc != 0)
5310                         return(rc);
5311
5312                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5313                 if (sb == NULL)
5314                         return (ENOMEM);
5315
5316                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5317                 rc = sbuf_finish(sb);
5318                 sbuf_delete(sb);
5319         } else {
5320                 char s[2];
5321                 int n;
5322
5323                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5324                 s[1] = 0;
5325
5326                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5327                 if (rc != 0)
5328                         return(rc);
5329
5330                 if (s[1] != 0)
5331                         return (EINVAL);
5332                 if (s[0] < '0' || s[0] > '9')
5333                         return (EINVAL);        /* not a number */
5334                 n = s[0] - '0';
5335                 if (n & ~(PAUSE_TX | PAUSE_RX))
5336                         return (EINVAL);        /* some other bit is set too */
5337
5338                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5339                 if (rc)
5340                         return (rc);
5341                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5342                         int link_ok = lc->link_ok;
5343
5344                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5345                         lc->requested_fc |= n;
5346                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5347                         lc->link_ok = link_ok;  /* restore */
5348                 }
5349                 end_synchronized_op(sc, 0);
5350         }
5351
5352         return (rc);
5353 }
5354
5355 static int
5356 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5357 {
5358         struct adapter *sc = arg1;
5359         int reg = arg2;
5360         uint64_t val;
5361
5362         val = t4_read_reg64(sc, reg);
5363
5364         return (sysctl_handle_64(oidp, &val, 0, req));
5365 }
5366
5367 static int
5368 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5369 {
5370         struct adapter *sc = arg1;
5371         int rc, t;
5372         uint32_t param, val;
5373
5374         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5375         if (rc)
5376                 return (rc);
5377         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5378             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5379             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5380         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5381         end_synchronized_op(sc, 0);
5382         if (rc)
5383                 return (rc);
5384
5385         /* unknown is returned as 0 but we display -1 in that case */
5386         t = val == 0 ? -1 : val;
5387
5388         rc = sysctl_handle_int(oidp, &t, 0, req);
5389         return (rc);
5390 }
5391
5392 #ifdef SBUF_DRAIN
5393 static int
5394 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5395 {
5396         struct adapter *sc = arg1;
5397         struct sbuf *sb;
5398         int rc, i;
5399         uint16_t incr[NMTUS][NCCTRL_WIN];
5400         static const char *dec_fac[] = {
5401                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5402                 "0.9375"
5403         };
5404
5405         rc = sysctl_wire_old_buffer(req, 0);
5406         if (rc != 0)
5407                 return (rc);
5408
5409         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5410         if (sb == NULL)
5411                 return (ENOMEM);
5412
5413         t4_read_cong_tbl(sc, incr);
5414
5415         for (i = 0; i < NCCTRL_WIN; ++i) {
5416                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5417                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5418                     incr[5][i], incr[6][i], incr[7][i]);
5419                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5420                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5421                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5422                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5423         }
5424
5425         rc = sbuf_finish(sb);
5426         sbuf_delete(sb);
5427
5428         return (rc);
5429 }
5430
5431 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5432         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5433         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5434         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5435 };
5436
5437 static int
5438 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5439 {
5440         struct adapter *sc = arg1;
5441         struct sbuf *sb;
5442         int rc, i, n, qid = arg2;
5443         uint32_t *buf, *p;
5444         char *qtype;
5445         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5446
5447         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5448             ("%s: bad qid %d\n", __func__, qid));
5449
5450         if (qid < CIM_NUM_IBQ) {
5451                 /* inbound queue */
5452                 qtype = "IBQ";
5453                 n = 4 * CIM_IBQ_SIZE;
5454                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5455                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5456         } else {
5457                 /* outbound queue */
5458                 qtype = "OBQ";
5459                 qid -= CIM_NUM_IBQ;
5460                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5461                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5462                 rc = t4_read_cim_obq(sc, qid, buf, n);
5463         }
5464
5465         if (rc < 0) {
5466                 rc = -rc;
5467                 goto done;
5468         }
5469         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5470
5471         rc = sysctl_wire_old_buffer(req, 0);
5472         if (rc != 0)
5473                 goto done;
5474
5475         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5476         if (sb == NULL) {
5477                 rc = ENOMEM;
5478                 goto done;
5479         }
5480
5481         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5482         for (i = 0, p = buf; i < n; i += 16, p += 4)
5483                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5484                     p[2], p[3]);
5485
5486         rc = sbuf_finish(sb);
5487         sbuf_delete(sb);
5488 done:
5489         free(buf, M_CXGBE);
5490         return (rc);
5491 }
5492
5493 static int
5494 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5495 {
5496         struct adapter *sc = arg1;
5497         u_int cfg;
5498         struct sbuf *sb;
5499         uint32_t *buf, *p;
5500         int rc;
5501
5502         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5503         if (rc != 0)
5504                 return (rc);
5505
5506         rc = sysctl_wire_old_buffer(req, 0);
5507         if (rc != 0)
5508                 return (rc);
5509
5510         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5511         if (sb == NULL)
5512                 return (ENOMEM);
5513
5514         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5515             M_ZERO | M_WAITOK);
5516
5517         rc = -t4_cim_read_la(sc, buf, NULL);
5518         if (rc != 0)
5519                 goto done;
5520
5521         sbuf_printf(sb, "Status   Data      PC%s",
5522             cfg & F_UPDBGLACAPTPCONLY ? "" :
5523             "     LS0Stat  LS0Addr             LS0Data");
5524
5525         KASSERT((sc->params.cim_la_size & 7) == 0,
5526             ("%s: p will walk off the end of buf", __func__));
5527
5528         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5529                 if (cfg & F_UPDBGLACAPTPCONLY) {
5530                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5531                             p[6], p[7]);
5532                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5533                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5534                             p[4] & 0xff, p[5] >> 8);
5535                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5536                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5537                             p[1] & 0xf, p[2] >> 4);
5538                 } else {
5539                         sbuf_printf(sb,
5540                             "\n  %02x   %x%07x %x%07x %08x %08x "
5541                             "%08x%08x%08x%08x",
5542                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5543                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5544                             p[6], p[7]);
5545                 }
5546         }
5547
5548         rc = sbuf_finish(sb);
5549         sbuf_delete(sb);
5550 done:
5551         free(buf, M_CXGBE);
5552         return (rc);
5553 }
5554
5555 static int
5556 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5557 {
5558         struct adapter *sc = arg1;
5559         u_int i;
5560         struct sbuf *sb;
5561         uint32_t *buf, *p;
5562         int rc;
5563
5564         rc = sysctl_wire_old_buffer(req, 0);
5565         if (rc != 0)
5566                 return (rc);
5567
5568         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5569         if (sb == NULL)
5570                 return (ENOMEM);
5571
5572         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5573             M_ZERO | M_WAITOK);
5574
5575         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5576         p = buf;
5577
5578         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5579                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5580                     p[1], p[0]);
5581         }
5582
5583         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5584         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5585                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5586                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5587                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5588                     (p[1] >> 2) | ((p[2] & 3) << 30),
5589                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5590                     p[0] & 1);
5591         }
5592
5593         rc = sbuf_finish(sb);
5594         sbuf_delete(sb);
5595         free(buf, M_CXGBE);
5596         return (rc);
5597 }
5598
5599 static int
5600 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5601 {
5602         struct adapter *sc = arg1;
5603         u_int i;
5604         struct sbuf *sb;
5605         uint32_t *buf, *p;
5606         int rc;
5607
5608         rc = sysctl_wire_old_buffer(req, 0);
5609         if (rc != 0)
5610                 return (rc);
5611
5612         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5613         if (sb == NULL)
5614                 return (ENOMEM);
5615
5616         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5617             M_ZERO | M_WAITOK);
5618
5619         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5620         p = buf;
5621
5622         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5623         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5624                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5625                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5626                     p[4], p[3], p[2], p[1], p[0]);
5627         }
5628
5629         sbuf_printf(sb, "\n\nCntl ID               Data");
5630         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5631                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5632                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5633         }
5634
5635         rc = sbuf_finish(sb);
5636         sbuf_delete(sb);
5637         free(buf, M_CXGBE);
5638         return (rc);
5639 }
5640
5641 static int
5642 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5643 {
5644         struct adapter *sc = arg1;
5645         struct sbuf *sb;
5646         int rc, i;
5647         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5648         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5649         uint16_t thres[CIM_NUM_IBQ];
5650         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5651         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5652         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5653
5654         if (is_t4(sc)) {
5655                 cim_num_obq = CIM_NUM_OBQ;
5656                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5657                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5658         } else {
5659                 cim_num_obq = CIM_NUM_OBQ_T5;
5660                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5661                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5662         }
5663         nq = CIM_NUM_IBQ + cim_num_obq;
5664
5665         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5666         if (rc == 0)
5667                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5668         if (rc != 0)
5669                 return (rc);
5670
5671         t4_read_cimq_cfg(sc, base, size, thres);
5672
5673         rc = sysctl_wire_old_buffer(req, 0);
5674         if (rc != 0)
5675                 return (rc);
5676
5677         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5678         if (sb == NULL)
5679                 return (ENOMEM);
5680
5681         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5682
5683         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5684                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5685                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5686                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5687                     G_QUEREMFLITS(p[2]) * 16);
5688         for ( ; i < nq; i++, p += 4, wr += 2)
5689                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5690                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5691                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5692                     G_QUEREMFLITS(p[2]) * 16);
5693
5694         rc = sbuf_finish(sb);
5695         sbuf_delete(sb);
5696
5697         return (rc);
5698 }
5699
5700 static int
5701 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5702 {
5703         struct adapter *sc = arg1;
5704         struct sbuf *sb;
5705         int rc;
5706         struct tp_cpl_stats stats;
5707
5708         rc = sysctl_wire_old_buffer(req, 0);
5709         if (rc != 0)
5710                 return (rc);
5711
5712         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5713         if (sb == NULL)
5714                 return (ENOMEM);
5715
5716         t4_tp_get_cpl_stats(sc, &stats);
5717
5718         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5719             "channel 3\n");
5720         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5721                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5722         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5723                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5724
5725         rc = sbuf_finish(sb);
5726         sbuf_delete(sb);
5727
5728         return (rc);
5729 }
5730
5731 static int
5732 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5733 {
5734         struct adapter *sc = arg1;
5735         struct sbuf *sb;
5736         int rc;
5737         struct tp_usm_stats stats;
5738
5739         rc = sysctl_wire_old_buffer(req, 0);
5740         if (rc != 0)
5741                 return(rc);
5742
5743         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5744         if (sb == NULL)
5745                 return (ENOMEM);
5746
5747         t4_get_usm_stats(sc, &stats);
5748
5749         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5750         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5751         sbuf_printf(sb, "Drops:  %u", stats.drops);
5752
5753         rc = sbuf_finish(sb);
5754         sbuf_delete(sb);
5755
5756         return (rc);
5757 }
5758
5759 const char *devlog_level_strings[] = {
5760         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5761         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5762         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5763         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5764         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5765         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5766 };
5767
5768 const char *devlog_facility_strings[] = {
5769         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5770         [FW_DEVLOG_FACILITY_CF]         = "CF",
5771         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5772         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5773         [FW_DEVLOG_FACILITY_RES]        = "RES",
5774         [FW_DEVLOG_FACILITY_HW]         = "HW",
5775         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5776         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5777         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5778         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5779         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5780         [FW_DEVLOG_FACILITY_VI]         = "VI",
5781         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5782         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5783         [FW_DEVLOG_FACILITY_TM]         = "TM",
5784         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5785         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5786         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5787         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5788         [FW_DEVLOG_FACILITY_RI]         = "RI",
5789         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5790         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5791         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5792         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5793 };
5794
5795 static int
5796 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5797 {
5798         struct adapter *sc = arg1;
5799         struct devlog_params *dparams = &sc->params.devlog;
5800         struct fw_devlog_e *buf, *e;
5801         int i, j, rc, nentries, first = 0, m;
5802         struct sbuf *sb;
5803         uint64_t ftstamp = UINT64_MAX;
5804
5805         if (dparams->start == 0) {
5806                 dparams->memtype = FW_MEMTYPE_EDC0;
5807                 dparams->start = 0x84000;
5808                 dparams->size = 32768;
5809         }
5810
5811         nentries = dparams->size / sizeof(struct fw_devlog_e);
5812
5813         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5814         if (buf == NULL)
5815                 return (ENOMEM);
5816
5817         m = fwmtype_to_hwmtype(dparams->memtype);
5818         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5819         if (rc != 0)
5820                 goto done;
5821
5822         for (i = 0; i < nentries; i++) {
5823                 e = &buf[i];
5824
5825                 if (e->timestamp == 0)
5826                         break;  /* end */
5827
5828                 e->timestamp = be64toh(e->timestamp);
5829                 e->seqno = be32toh(e->seqno);
5830                 for (j = 0; j < 8; j++)
5831                         e->params[j] = be32toh(e->params[j]);
5832
5833                 if (e->timestamp < ftstamp) {
5834                         ftstamp = e->timestamp;
5835                         first = i;
5836                 }
5837         }
5838
5839         if (buf[first].timestamp == 0)
5840                 goto done;      /* nothing in the log */
5841
5842         rc = sysctl_wire_old_buffer(req, 0);
5843         if (rc != 0)
5844                 goto done;
5845
5846         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5847         if (sb == NULL) {
5848                 rc = ENOMEM;
5849                 goto done;
5850         }
5851         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5852             "Seq#", "Tstamp", "Level", "Facility", "Message");
5853
5854         i = first;
5855         do {
5856                 e = &buf[i];
5857                 if (e->timestamp == 0)
5858                         break;  /* end */
5859
5860                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5861                     e->seqno, e->timestamp,
5862                     (e->level < nitems(devlog_level_strings) ?
5863                         devlog_level_strings[e->level] : "UNKNOWN"),
5864                     (e->facility < nitems(devlog_facility_strings) ?
5865                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5866                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5867                     e->params[2], e->params[3], e->params[4],
5868                     e->params[5], e->params[6], e->params[7]);
5869
5870                 if (++i == nentries)
5871                         i = 0;
5872         } while (i != first);
5873
5874         rc = sbuf_finish(sb);
5875         sbuf_delete(sb);
5876 done:
5877         free(buf, M_CXGBE);
5878         return (rc);
5879 }
5880
5881 static int
5882 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5883 {
5884         struct adapter *sc = arg1;
5885         struct sbuf *sb;
5886         int rc;
5887         struct tp_fcoe_stats stats[4];
5888
5889         rc = sysctl_wire_old_buffer(req, 0);
5890         if (rc != 0)
5891                 return (rc);
5892
5893         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5894         if (sb == NULL)
5895                 return (ENOMEM);
5896
5897         t4_get_fcoe_stats(sc, 0, &stats[0]);
5898         t4_get_fcoe_stats(sc, 1, &stats[1]);
5899         t4_get_fcoe_stats(sc, 2, &stats[2]);
5900         t4_get_fcoe_stats(sc, 3, &stats[3]);
5901
5902         sbuf_printf(sb, "                   channel 0        channel 1        "
5903             "channel 2        channel 3\n");
5904         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5905             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5906             stats[3].octetsDDP);
5907         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5908             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5909         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5910             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5911             stats[3].framesDrop);
5912
5913         rc = sbuf_finish(sb);
5914         sbuf_delete(sb);
5915
5916         return (rc);
5917 }
5918
5919 static int
5920 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5921 {
5922         struct adapter *sc = arg1;
5923         struct sbuf *sb;
5924         int rc, i;
5925         unsigned int map, kbps, ipg, mode;
5926         unsigned int pace_tab[NTX_SCHED];
5927
5928         rc = sysctl_wire_old_buffer(req, 0);
5929         if (rc != 0)
5930                 return (rc);
5931
5932         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5933         if (sb == NULL)
5934                 return (ENOMEM);
5935
5936         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5937         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5938         t4_read_pace_tbl(sc, pace_tab);
5939
5940         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5941             "Class IPG (0.1 ns)   Flow IPG (us)");
5942
5943         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5944                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5945                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5946                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5947                 if (kbps)
5948                         sbuf_printf(sb, "%9u     ", kbps);
5949                 else
5950                         sbuf_printf(sb, " disabled     ");
5951
5952                 if (ipg)
5953                         sbuf_printf(sb, "%13u        ", ipg);
5954                 else
5955                         sbuf_printf(sb, "     disabled        ");
5956
5957                 if (pace_tab[i])
5958                         sbuf_printf(sb, "%10u", pace_tab[i]);
5959                 else
5960                         sbuf_printf(sb, "  disabled");
5961         }
5962
5963         rc = sbuf_finish(sb);
5964         sbuf_delete(sb);
5965
5966         return (rc);
5967 }
5968
5969 static int
5970 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5971 {
5972         struct adapter *sc = arg1;
5973         struct sbuf *sb;
5974         int rc, i, j;
5975         uint64_t *p0, *p1;
5976         struct lb_port_stats s[2];
5977         static const char *stat_name[] = {
5978                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5979                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5980                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5981                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5982                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5983                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5984                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5985         };
5986
5987         rc = sysctl_wire_old_buffer(req, 0);
5988         if (rc != 0)
5989                 return (rc);
5990
5991         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5992         if (sb == NULL)
5993                 return (ENOMEM);
5994
5995         memset(s, 0, sizeof(s));
5996
5997         for (i = 0; i < 4; i += 2) {
5998                 t4_get_lb_stats(sc, i, &s[0]);
5999                 t4_get_lb_stats(sc, i + 1, &s[1]);
6000
6001                 p0 = &s[0].octets;
6002                 p1 = &s[1].octets;
6003                 sbuf_printf(sb, "%s                       Loopback %u"
6004                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6005
6006                 for (j = 0; j < nitems(stat_name); j++)
6007                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6008                                    *p0++, *p1++);
6009         }
6010
6011         rc = sbuf_finish(sb);
6012         sbuf_delete(sb);
6013
6014         return (rc);
6015 }
6016
6017 static int
6018 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6019 {
6020         int rc = 0;
6021         struct port_info *pi = arg1;
6022         struct sbuf *sb;
6023         static const char *linkdnreasons[] = {
6024                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6025                 "PHY overheated", "unknown", "rx los", "reserved7"
6026         };
6027
6028         rc = sysctl_wire_old_buffer(req, 0);
6029         if (rc != 0)
6030                 return(rc);
6031         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6032         if (sb == NULL)
6033                 return (ENOMEM);
6034
6035         if (pi->linkdnrc < 0)
6036                 sbuf_printf(sb, "n/a");
6037         else if (pi->linkdnrc < nitems(linkdnreasons))
6038                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6039         else
6040                 sbuf_printf(sb, "%d", pi->linkdnrc);
6041
6042         rc = sbuf_finish(sb);
6043         sbuf_delete(sb);
6044
6045         return (rc);
6046 }
6047
6048 struct mem_desc {
6049         unsigned int base;
6050         unsigned int limit;
6051         unsigned int idx;
6052 };
6053
6054 static int
6055 mem_desc_cmp(const void *a, const void *b)
6056 {
6057         return ((const struct mem_desc *)a)->base -
6058                ((const struct mem_desc *)b)->base;
6059 }
6060
6061 static void
6062 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6063     unsigned int to)
6064 {
6065         unsigned int size;
6066
6067         size = to - from + 1;
6068         if (size == 0)
6069                 return;
6070
6071         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6072         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6073 }
6074
6075 static int
6076 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6077 {
6078         struct adapter *sc = arg1;
6079         struct sbuf *sb;
6080         int rc, i, n;
6081         uint32_t lo, hi, used, alloc;
6082         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6083         static const char *region[] = {
6084                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6085                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6086                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6087                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6088                 "RQUDP region:", "PBL region:", "TXPBL region:",
6089                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6090                 "On-chip queues:"
6091         };
6092         struct mem_desc avail[4];
6093         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6094         struct mem_desc *md = mem;
6095
6096         rc = sysctl_wire_old_buffer(req, 0);
6097         if (rc != 0)
6098                 return (rc);
6099
6100         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6101         if (sb == NULL)
6102                 return (ENOMEM);
6103
6104         for (i = 0; i < nitems(mem); i++) {
6105                 mem[i].limit = 0;
6106                 mem[i].idx = i;
6107         }
6108
6109         /* Find and sort the populated memory ranges */
6110         i = 0;
6111         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6112         if (lo & F_EDRAM0_ENABLE) {
6113                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6114                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6115                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6116                 avail[i].idx = 0;
6117                 i++;
6118         }
6119         if (lo & F_EDRAM1_ENABLE) {
6120                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6121                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6122                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6123                 avail[i].idx = 1;
6124                 i++;
6125         }
6126         if (lo & F_EXT_MEM_ENABLE) {
6127                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6128                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6129                 avail[i].limit = avail[i].base +
6130                     (G_EXT_MEM_SIZE(hi) << 20);
6131                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6132                 i++;
6133         }
6134         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6135                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6136                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6137                 avail[i].limit = avail[i].base +
6138                     (G_EXT_MEM1_SIZE(hi) << 20);
6139                 avail[i].idx = 4;
6140                 i++;
6141         }
6142         if (!i)                                    /* no memory available */
6143                 return 0;
6144         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6145
6146         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6147         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6148         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6149         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6150         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6151         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6152         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6153         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6154         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6155
6156         /* the next few have explicit upper bounds */
6157         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6158         md->limit = md->base - 1 +
6159                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6160                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6161         md++;
6162
6163         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6164         md->limit = md->base - 1 +
6165                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6166                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6167         md++;
6168
6169         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6170                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6171                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6172                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6173         } else {
6174                 md->base = 0;
6175                 md->idx = nitems(region);  /* hide it */
6176         }
6177         md++;
6178
6179 #define ulp_region(reg) \
6180         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6181         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6182
6183         ulp_region(RX_ISCSI);
6184         ulp_region(RX_TDDP);
6185         ulp_region(TX_TPT);
6186         ulp_region(RX_STAG);
6187         ulp_region(RX_RQ);
6188         ulp_region(RX_RQUDP);
6189         ulp_region(RX_PBL);
6190         ulp_region(TX_PBL);
6191 #undef ulp_region
6192
6193         md->base = 0;
6194         md->idx = nitems(region);
6195         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6196                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6197                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6198                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6199         }
6200         md++;
6201
6202         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6203         md->limit = md->base + sc->tids.ntids - 1;
6204         md++;
6205         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6206         md->limit = md->base + sc->tids.ntids - 1;
6207         md++;
6208
6209         md->base = sc->vres.ocq.start;
6210         if (sc->vres.ocq.size)
6211                 md->limit = md->base + sc->vres.ocq.size - 1;
6212         else
6213                 md->idx = nitems(region);  /* hide it */
6214         md++;
6215
6216         /* add any address-space holes, there can be up to 3 */
6217         for (n = 0; n < i - 1; n++)
6218                 if (avail[n].limit < avail[n + 1].base)
6219                         (md++)->base = avail[n].limit;
6220         if (avail[n].limit)
6221                 (md++)->base = avail[n].limit;
6222
6223         n = md - mem;
6224         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6225
6226         for (lo = 0; lo < i; lo++)
6227                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6228                                 avail[lo].limit - 1);
6229
6230         sbuf_printf(sb, "\n");
6231         for (i = 0; i < n; i++) {
6232                 if (mem[i].idx >= nitems(region))
6233                         continue;                        /* skip holes */
6234                 if (!mem[i].limit)
6235                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6236                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6237                                 mem[i].limit);
6238         }
6239
6240         sbuf_printf(sb, "\n");
6241         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6242         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6243         mem_region_show(sb, "uP RAM:", lo, hi);
6244
6245         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6246         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6247         mem_region_show(sb, "uP Extmem2:", lo, hi);
6248
6249         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6250         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6251                    G_PMRXMAXPAGE(lo),
6252                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6253                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6254
6255         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6256         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6257         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6258                    G_PMTXMAXPAGE(lo),
6259                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6260                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6261         sbuf_printf(sb, "%u p-structs\n",
6262                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6263
6264         for (i = 0; i < 4; i++) {
6265                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6266                 if (is_t4(sc)) {
6267                         used = G_USED(lo);
6268                         alloc = G_ALLOC(lo);
6269                 } else {
6270                         used = G_T5_USED(lo);
6271                         alloc = G_T5_ALLOC(lo);
6272                 }
6273                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6274                            i, used, alloc);
6275         }
6276         for (i = 0; i < 4; i++) {
6277                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6278                 if (is_t4(sc)) {
6279                         used = G_USED(lo);
6280                         alloc = G_ALLOC(lo);
6281                 } else {
6282                         used = G_T5_USED(lo);
6283                         alloc = G_T5_ALLOC(lo);
6284                 }
6285                 sbuf_printf(sb,
6286                            "\nLoopback %d using %u pages out of %u allocated",
6287                            i, used, alloc);
6288         }
6289
6290         rc = sbuf_finish(sb);
6291         sbuf_delete(sb);
6292
6293         return (rc);
6294 }
6295
6296 static inline void
6297 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6298 {
6299         *mask = x | y;
6300         y = htobe64(y);
6301         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6302 }
6303
6304 static int
6305 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6306 {
6307         struct adapter *sc = arg1;
6308         struct sbuf *sb;
6309         int rc, i, n;
6310
6311         rc = sysctl_wire_old_buffer(req, 0);
6312         if (rc != 0)
6313                 return (rc);
6314
6315         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6316         if (sb == NULL)
6317                 return (ENOMEM);
6318
6319         sbuf_printf(sb,
6320             "Idx  Ethernet address     Mask     Vld Ports PF"
6321             "  VF              Replication             P0 P1 P2 P3  ML");
6322         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6323             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6324         for (i = 0; i < n; i++) {
6325                 uint64_t tcamx, tcamy, mask;
6326                 uint32_t cls_lo, cls_hi;
6327                 uint8_t addr[ETHER_ADDR_LEN];
6328
6329                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6330                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6331                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6332                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6333
6334                 if (tcamx & tcamy)
6335                         continue;
6336
6337                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6338                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6339                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6340                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6341                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6342                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6343                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6344
6345                 if (cls_lo & F_REPLICATE) {
6346                         struct fw_ldst_cmd ldst_cmd;
6347
6348                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6349                         ldst_cmd.op_to_addrspace =
6350                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6351                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6352                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6353                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6354                         ldst_cmd.u.mps.fid_ctl =
6355                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6356                                 V_FW_LDST_CMD_CTL(i));
6357
6358                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6359                             "t4mps");
6360                         if (rc)
6361                                 break;
6362                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6363                             sizeof(ldst_cmd), &ldst_cmd);
6364                         end_synchronized_op(sc, 0);
6365
6366                         if (rc != 0) {
6367                                 sbuf_printf(sb,
6368                                     " ------------ error %3u ------------", rc);
6369                                 rc = 0;
6370                         } else {
6371                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6372                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6373                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6374                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6375                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6376                         }
6377                 } else
6378                         sbuf_printf(sb, "%36s", "");
6379
6380                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6381                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6382                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6383         }
6384
6385         if (rc)
6386                 (void) sbuf_finish(sb);
6387         else
6388                 rc = sbuf_finish(sb);
6389         sbuf_delete(sb);
6390
6391         return (rc);
6392 }
6393
6394 static int
6395 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6396 {
6397         struct adapter *sc = arg1;
6398         struct sbuf *sb;
6399         int rc;
6400         uint16_t mtus[NMTUS];
6401
6402         rc = sysctl_wire_old_buffer(req, 0);
6403         if (rc != 0)
6404                 return (rc);
6405
6406         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6407         if (sb == NULL)
6408                 return (ENOMEM);
6409
6410         t4_read_mtu_tbl(sc, mtus, NULL);
6411
6412         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6413             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6414             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6415             mtus[14], mtus[15]);
6416
6417         rc = sbuf_finish(sb);
6418         sbuf_delete(sb);
6419
6420         return (rc);
6421 }
6422
6423 static int
6424 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6425 {
6426         struct adapter *sc = arg1;
6427         struct sbuf *sb;
6428         int rc, i;
6429         uint32_t cnt[PM_NSTATS];
6430         uint64_t cyc[PM_NSTATS];
6431         static const char *rx_stats[] = {
6432                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6433         };
6434         static const char *tx_stats[] = {
6435                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6436         };
6437
6438         rc = sysctl_wire_old_buffer(req, 0);
6439         if (rc != 0)
6440                 return (rc);
6441
6442         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6443         if (sb == NULL)
6444                 return (ENOMEM);
6445
6446         t4_pmtx_get_stats(sc, cnt, cyc);
6447         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6448         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6449                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6450                     cyc[i]);
6451
6452         t4_pmrx_get_stats(sc, cnt, cyc);
6453         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6454         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6455                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6456                     cyc[i]);
6457
6458         rc = sbuf_finish(sb);
6459         sbuf_delete(sb);
6460
6461         return (rc);
6462 }
6463
6464 static int
6465 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6466 {
6467         struct adapter *sc = arg1;
6468         struct sbuf *sb;
6469         int rc;
6470         struct tp_rdma_stats stats;
6471
6472         rc = sysctl_wire_old_buffer(req, 0);
6473         if (rc != 0)
6474                 return (rc);
6475
6476         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6477         if (sb == NULL)
6478                 return (ENOMEM);
6479
6480         t4_tp_get_rdma_stats(sc, &stats);
6481         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6482         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6483
6484         rc = sbuf_finish(sb);
6485         sbuf_delete(sb);
6486
6487         return (rc);
6488 }
6489
6490 static int
6491 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6492 {
6493         struct adapter *sc = arg1;
6494         struct sbuf *sb;
6495         int rc;
6496         struct tp_tcp_stats v4, v6;
6497
6498         rc = sysctl_wire_old_buffer(req, 0);
6499         if (rc != 0)
6500                 return (rc);
6501
6502         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6503         if (sb == NULL)
6504                 return (ENOMEM);
6505
6506         t4_tp_get_tcp_stats(sc, &v4, &v6);
6507         sbuf_printf(sb,
6508             "                                IP                 IPv6\n");
6509         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6510             v4.tcpOutRsts, v6.tcpOutRsts);
6511         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6512             v4.tcpInSegs, v6.tcpInSegs);
6513         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6514             v4.tcpOutSegs, v6.tcpOutSegs);
6515         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6516             v4.tcpRetransSegs, v6.tcpRetransSegs);
6517
6518         rc = sbuf_finish(sb);
6519         sbuf_delete(sb);
6520
6521         return (rc);
6522 }
6523
6524 static int
6525 sysctl_tids(SYSCTL_HANDLER_ARGS)
6526 {
6527         struct adapter *sc = arg1;
6528         struct sbuf *sb;
6529         int rc;
6530         struct tid_info *t = &sc->tids;
6531
6532         rc = sysctl_wire_old_buffer(req, 0);
6533         if (rc != 0)
6534                 return (rc);
6535
6536         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6537         if (sb == NULL)
6538                 return (ENOMEM);
6539
6540         if (t->natids) {
6541                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6542                     t->atids_in_use);
6543         }
6544
6545         if (t->ntids) {
6546                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6547                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6548
6549                         if (b) {
6550                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6551                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6552                                     t->ntids - 1);
6553                         } else {
6554                                 sbuf_printf(sb, "TID range: %u-%u",
6555                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6556                                     t->ntids - 1);
6557                         }
6558                 } else
6559                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6560                 sbuf_printf(sb, ", in use: %u\n",
6561                     atomic_load_acq_int(&t->tids_in_use));
6562         }
6563
6564         if (t->nstids) {
6565                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6566                     t->stid_base + t->nstids - 1, t->stids_in_use);
6567         }
6568
6569         if (t->nftids) {
6570                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6571                     t->ftid_base + t->nftids - 1);
6572         }
6573
6574         if (t->netids) {
6575                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6576                     t->etid_base + t->netids - 1);
6577         }
6578
6579         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6580             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6581             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6582
6583         rc = sbuf_finish(sb);
6584         sbuf_delete(sb);
6585
6586         return (rc);
6587 }
6588
6589 static int
6590 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6591 {
6592         struct adapter *sc = arg1;
6593         struct sbuf *sb;
6594         int rc;
6595         struct tp_err_stats stats;
6596
6597         rc = sysctl_wire_old_buffer(req, 0);
6598         if (rc != 0)
6599                 return (rc);
6600
6601         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6602         if (sb == NULL)
6603                 return (ENOMEM);
6604
6605         t4_tp_get_err_stats(sc, &stats);
6606
6607         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6608                       "channel 3\n");
6609         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6610             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6611             stats.macInErrs[3]);
6612         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6613             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6614             stats.hdrInErrs[3]);
6615         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6616             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6617             stats.tcpInErrs[3]);
6618         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6619             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6620             stats.tcp6InErrs[3]);
6621         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6622             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6623             stats.tnlCongDrops[3]);
6624         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6625             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6626             stats.tnlTxDrops[3]);
6627         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6628             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6629             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6630         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6631             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6632             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6633         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6634             stats.ofldNoNeigh, stats.ofldCongDefer);
6635
6636         rc = sbuf_finish(sb);
6637         sbuf_delete(sb);
6638
6639         return (rc);
6640 }
6641
6642 struct field_desc {
6643         const char *name;
6644         u_int start;
6645         u_int width;
6646 };
6647
6648 static void
6649 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6650 {
6651         char buf[32];
6652         int line_size = 0;
6653
6654         while (f->name) {
6655                 uint64_t mask = (1ULL << f->width) - 1;
6656                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6657                     ((uintmax_t)v >> f->start) & mask);
6658
6659                 if (line_size + len >= 79) {
6660                         line_size = 8;
6661                         sbuf_printf(sb, "\n        ");
6662                 }
6663                 sbuf_printf(sb, "%s ", buf);
6664                 line_size += len + 1;
6665                 f++;
6666         }
6667         sbuf_printf(sb, "\n");
6668 }
6669
6670 static struct field_desc tp_la0[] = {
6671         { "RcfOpCodeOut", 60, 4 },
6672         { "State", 56, 4 },
6673         { "WcfState", 52, 4 },
6674         { "RcfOpcSrcOut", 50, 2 },
6675         { "CRxError", 49, 1 },
6676         { "ERxError", 48, 1 },
6677         { "SanityFailed", 47, 1 },
6678         { "SpuriousMsg", 46, 1 },
6679         { "FlushInputMsg", 45, 1 },
6680         { "FlushInputCpl", 44, 1 },
6681         { "RssUpBit", 43, 1 },
6682         { "RssFilterHit", 42, 1 },
6683         { "Tid", 32, 10 },
6684         { "InitTcb", 31, 1 },
6685         { "LineNumber", 24, 7 },
6686         { "Emsg", 23, 1 },
6687         { "EdataOut", 22, 1 },
6688         { "Cmsg", 21, 1 },
6689         { "CdataOut", 20, 1 },
6690         { "EreadPdu", 19, 1 },
6691         { "CreadPdu", 18, 1 },
6692         { "TunnelPkt", 17, 1 },
6693         { "RcfPeerFin", 16, 1 },
6694         { "RcfReasonOut", 12, 4 },
6695         { "TxCchannel", 10, 2 },
6696         { "RcfTxChannel", 8, 2 },
6697         { "RxEchannel", 6, 2 },
6698         { "RcfRxChannel", 5, 1 },
6699         { "RcfDataOutSrdy", 4, 1 },
6700         { "RxDvld", 3, 1 },
6701         { "RxOoDvld", 2, 1 },
6702         { "RxCongestion", 1, 1 },
6703         { "TxCongestion", 0, 1 },
6704         { NULL }
6705 };
6706
6707 static struct field_desc tp_la1[] = {
6708         { "CplCmdIn", 56, 8 },
6709         { "CplCmdOut", 48, 8 },
6710         { "ESynOut", 47, 1 },
6711         { "EAckOut", 46, 1 },
6712         { "EFinOut", 45, 1 },
6713         { "ERstOut", 44, 1 },
6714         { "SynIn", 43, 1 },
6715         { "AckIn", 42, 1 },
6716         { "FinIn", 41, 1 },
6717         { "RstIn", 40, 1 },
6718         { "DataIn", 39, 1 },
6719         { "DataInVld", 38, 1 },
6720         { "PadIn", 37, 1 },
6721         { "RxBufEmpty", 36, 1 },
6722         { "RxDdp", 35, 1 },
6723         { "RxFbCongestion", 34, 1 },
6724         { "TxFbCongestion", 33, 1 },
6725         { "TxPktSumSrdy", 32, 1 },
6726         { "RcfUlpType", 28, 4 },
6727         { "Eread", 27, 1 },
6728         { "Ebypass", 26, 1 },
6729         { "Esave", 25, 1 },
6730         { "Static0", 24, 1 },
6731         { "Cread", 23, 1 },
6732         { "Cbypass", 22, 1 },
6733         { "Csave", 21, 1 },
6734         { "CPktOut", 20, 1 },
6735         { "RxPagePoolFull", 18, 2 },
6736         { "RxLpbkPkt", 17, 1 },
6737         { "TxLpbkPkt", 16, 1 },
6738         { "RxVfValid", 15, 1 },
6739         { "SynLearned", 14, 1 },
6740         { "SetDelEntry", 13, 1 },
6741         { "SetInvEntry", 12, 1 },
6742         { "CpcmdDvld", 11, 1 },
6743         { "CpcmdSave", 10, 1 },
6744         { "RxPstructsFull", 8, 2 },
6745         { "EpcmdDvld", 7, 1 },
6746         { "EpcmdFlush", 6, 1 },
6747         { "EpcmdTrimPrefix", 5, 1 },
6748         { "EpcmdTrimPostfix", 4, 1 },
6749         { "ERssIp4Pkt", 3, 1 },
6750         { "ERssIp6Pkt", 2, 1 },
6751         { "ERssTcpUdpPkt", 1, 1 },
6752         { "ERssFceFipPkt", 0, 1 },
6753         { NULL }
6754 };
6755
6756 static struct field_desc tp_la2[] = {
6757         { "CplCmdIn", 56, 8 },
6758         { "MpsVfVld", 55, 1 },
6759         { "MpsPf", 52, 3 },
6760         { "MpsVf", 44, 8 },
6761         { "SynIn", 43, 1 },
6762         { "AckIn", 42, 1 },
6763         { "FinIn", 41, 1 },
6764         { "RstIn", 40, 1 },
6765         { "DataIn", 39, 1 },
6766         { "DataInVld", 38, 1 },
6767         { "PadIn", 37, 1 },
6768         { "RxBufEmpty", 36, 1 },
6769         { "RxDdp", 35, 1 },
6770         { "RxFbCongestion", 34, 1 },
6771         { "TxFbCongestion", 33, 1 },
6772         { "TxPktSumSrdy", 32, 1 },
6773         { "RcfUlpType", 28, 4 },
6774         { "Eread", 27, 1 },
6775         { "Ebypass", 26, 1 },
6776         { "Esave", 25, 1 },
6777         { "Static0", 24, 1 },
6778         { "Cread", 23, 1 },
6779         { "Cbypass", 22, 1 },
6780         { "Csave", 21, 1 },
6781         { "CPktOut", 20, 1 },
6782         { "RxPagePoolFull", 18, 2 },
6783         { "RxLpbkPkt", 17, 1 },
6784         { "TxLpbkPkt", 16, 1 },
6785         { "RxVfValid", 15, 1 },
6786         { "SynLearned", 14, 1 },
6787         { "SetDelEntry", 13, 1 },
6788         { "SetInvEntry", 12, 1 },
6789         { "CpcmdDvld", 11, 1 },
6790         { "CpcmdSave", 10, 1 },
6791         { "RxPstructsFull", 8, 2 },
6792         { "EpcmdDvld", 7, 1 },
6793         { "EpcmdFlush", 6, 1 },
6794         { "EpcmdTrimPrefix", 5, 1 },
6795         { "EpcmdTrimPostfix", 4, 1 },
6796         { "ERssIp4Pkt", 3, 1 },
6797         { "ERssIp6Pkt", 2, 1 },
6798         { "ERssTcpUdpPkt", 1, 1 },
6799         { "ERssFceFipPkt", 0, 1 },
6800         { NULL }
6801 };
6802
6803 static void
6804 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6805 {
6806
6807         field_desc_show(sb, *p, tp_la0);
6808 }
6809
6810 static void
6811 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6812 {
6813
6814         if (idx)
6815                 sbuf_printf(sb, "\n");
6816         field_desc_show(sb, p[0], tp_la0);
6817         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6818                 field_desc_show(sb, p[1], tp_la0);
6819 }
6820
6821 static void
6822 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6823 {
6824
6825         if (idx)
6826                 sbuf_printf(sb, "\n");
6827         field_desc_show(sb, p[0], tp_la0);
6828         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6829                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6830 }
6831
6832 static int
6833 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6834 {
6835         struct adapter *sc = arg1;
6836         struct sbuf *sb;
6837         uint64_t *buf, *p;
6838         int rc;
6839         u_int i, inc;
6840         void (*show_func)(struct sbuf *, uint64_t *, int);
6841
6842         rc = sysctl_wire_old_buffer(req, 0);
6843         if (rc != 0)
6844                 return (rc);
6845
6846         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6847         if (sb == NULL)
6848                 return (ENOMEM);
6849
6850         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6851
6852         t4_tp_read_la(sc, buf, NULL);
6853         p = buf;
6854
6855         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6856         case 2:
6857                 inc = 2;
6858                 show_func = tp_la_show2;
6859                 break;
6860         case 3:
6861                 inc = 2;
6862                 show_func = tp_la_show3;
6863                 break;
6864         default:
6865                 inc = 1;
6866                 show_func = tp_la_show;
6867         }
6868
6869         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6870                 (*show_func)(sb, p, i);
6871
6872         rc = sbuf_finish(sb);
6873         sbuf_delete(sb);
6874         free(buf, M_CXGBE);
6875         return (rc);
6876 }
6877
6878 static int
6879 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6880 {
6881         struct adapter *sc = arg1;
6882         struct sbuf *sb;
6883         int rc;
6884         u64 nrate[NCHAN], orate[NCHAN];
6885
6886         rc = sysctl_wire_old_buffer(req, 0);
6887         if (rc != 0)
6888                 return (rc);
6889
6890         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6891         if (sb == NULL)
6892                 return (ENOMEM);
6893
6894         t4_get_chan_txrate(sc, nrate, orate);
6895         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6896                  "channel 3\n");
6897         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6898             nrate[0], nrate[1], nrate[2], nrate[3]);
6899         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6900             orate[0], orate[1], orate[2], orate[3]);
6901
6902         rc = sbuf_finish(sb);
6903         sbuf_delete(sb);
6904
6905         return (rc);
6906 }
6907
6908 static int
6909 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6910 {
6911         struct adapter *sc = arg1;
6912         struct sbuf *sb;
6913         uint32_t *buf, *p;
6914         int rc, i;
6915
6916         rc = sysctl_wire_old_buffer(req, 0);
6917         if (rc != 0)
6918                 return (rc);
6919
6920         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6921         if (sb == NULL)
6922                 return (ENOMEM);
6923
6924         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6925             M_ZERO | M_WAITOK);
6926
6927         t4_ulprx_read_la(sc, buf);
6928         p = buf;
6929
6930         sbuf_printf(sb, "      Pcmd        Type   Message"
6931             "                Data");
6932         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6933                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6934                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6935         }
6936
6937         rc = sbuf_finish(sb);
6938         sbuf_delete(sb);
6939         free(buf, M_CXGBE);
6940         return (rc);
6941 }
6942
6943 static int
6944 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6945 {
6946         struct adapter *sc = arg1;
6947         struct sbuf *sb;
6948         int rc, v;
6949
6950         rc = sysctl_wire_old_buffer(req, 0);
6951         if (rc != 0)
6952                 return (rc);
6953
6954         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6955         if (sb == NULL)
6956                 return (ENOMEM);
6957
6958         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6959         if (G_STATSOURCE_T5(v) == 7) {
6960                 if (G_STATMODE(v) == 0) {
6961                         sbuf_printf(sb, "total %d, incomplete %d",
6962                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6963                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6964                 } else if (G_STATMODE(v) == 1) {
6965                         sbuf_printf(sb, "total %d, data overflow %d",
6966                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6967                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6968                 }
6969         }
6970         rc = sbuf_finish(sb);
6971         sbuf_delete(sb);
6972
6973         return (rc);
6974 }
6975 #endif
6976
6977 static uint32_t
6978 fconf_to_mode(uint32_t fconf)
6979 {
6980         uint32_t mode;
6981
6982         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6983             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6984
6985         if (fconf & F_FRAGMENTATION)
6986                 mode |= T4_FILTER_IP_FRAGMENT;
6987
6988         if (fconf & F_MPSHITTYPE)
6989                 mode |= T4_FILTER_MPS_HIT_TYPE;
6990
6991         if (fconf & F_MACMATCH)
6992                 mode |= T4_FILTER_MAC_IDX;
6993
6994         if (fconf & F_ETHERTYPE)
6995                 mode |= T4_FILTER_ETH_TYPE;
6996
6997         if (fconf & F_PROTOCOL)
6998                 mode |= T4_FILTER_IP_PROTO;
6999
7000         if (fconf & F_TOS)
7001                 mode |= T4_FILTER_IP_TOS;
7002
7003         if (fconf & F_VLAN)
7004                 mode |= T4_FILTER_VLAN;
7005
7006         if (fconf & F_VNIC_ID)
7007                 mode |= T4_FILTER_VNIC;
7008
7009         if (fconf & F_PORT)
7010                 mode |= T4_FILTER_PORT;
7011
7012         if (fconf & F_FCOE)
7013                 mode |= T4_FILTER_FCoE;
7014
7015         return (mode);
7016 }
7017
7018 static uint32_t
7019 mode_to_fconf(uint32_t mode)
7020 {
7021         uint32_t fconf = 0;
7022
7023         if (mode & T4_FILTER_IP_FRAGMENT)
7024                 fconf |= F_FRAGMENTATION;
7025
7026         if (mode & T4_FILTER_MPS_HIT_TYPE)
7027                 fconf |= F_MPSHITTYPE;
7028
7029         if (mode & T4_FILTER_MAC_IDX)
7030                 fconf |= F_MACMATCH;
7031
7032         if (mode & T4_FILTER_ETH_TYPE)
7033                 fconf |= F_ETHERTYPE;
7034
7035         if (mode & T4_FILTER_IP_PROTO)
7036                 fconf |= F_PROTOCOL;
7037
7038         if (mode & T4_FILTER_IP_TOS)
7039                 fconf |= F_TOS;
7040
7041         if (mode & T4_FILTER_VLAN)
7042                 fconf |= F_VLAN;
7043
7044         if (mode & T4_FILTER_VNIC)
7045                 fconf |= F_VNIC_ID;
7046
7047         if (mode & T4_FILTER_PORT)
7048                 fconf |= F_PORT;
7049
7050         if (mode & T4_FILTER_FCoE)
7051                 fconf |= F_FCOE;
7052
7053         return (fconf);
7054 }
7055
7056 static uint32_t
7057 fspec_to_fconf(struct t4_filter_specification *fs)
7058 {
7059         uint32_t fconf = 0;
7060
7061         if (fs->val.frag || fs->mask.frag)
7062                 fconf |= F_FRAGMENTATION;
7063
7064         if (fs->val.matchtype || fs->mask.matchtype)
7065                 fconf |= F_MPSHITTYPE;
7066
7067         if (fs->val.macidx || fs->mask.macidx)
7068                 fconf |= F_MACMATCH;
7069
7070         if (fs->val.ethtype || fs->mask.ethtype)
7071                 fconf |= F_ETHERTYPE;
7072
7073         if (fs->val.proto || fs->mask.proto)
7074                 fconf |= F_PROTOCOL;
7075
7076         if (fs->val.tos || fs->mask.tos)
7077                 fconf |= F_TOS;
7078
7079         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7080                 fconf |= F_VLAN;
7081
7082         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7083                 fconf |= F_VNIC_ID;
7084
7085         if (fs->val.iport || fs->mask.iport)
7086                 fconf |= F_PORT;
7087
7088         if (fs->val.fcoe || fs->mask.fcoe)
7089                 fconf |= F_FCOE;
7090
7091         return (fconf);
7092 }
7093
7094 static int
7095 get_filter_mode(struct adapter *sc, uint32_t *mode)
7096 {
7097         int rc;
7098         uint32_t fconf;
7099
7100         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7101             "t4getfm");
7102         if (rc)
7103                 return (rc);
7104
7105         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7106             A_TP_VLAN_PRI_MAP);
7107
7108         if (sc->params.tp.vlan_pri_map != fconf) {
7109                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7110                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7111                     fconf);
7112                 sc->params.tp.vlan_pri_map = fconf;
7113         }
7114
7115         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7116
7117         end_synchronized_op(sc, LOCK_HELD);
7118         return (0);
7119 }
7120
7121 static int
7122 set_filter_mode(struct adapter *sc, uint32_t mode)
7123 {
7124         uint32_t fconf;
7125         int rc;
7126
7127         fconf = mode_to_fconf(mode);
7128
7129         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7130             "t4setfm");
7131         if (rc)
7132                 return (rc);
7133
7134         if (sc->tids.ftids_in_use > 0) {
7135                 rc = EBUSY;
7136                 goto done;
7137         }
7138
7139 #ifdef TCP_OFFLOAD
7140         if (sc->offload_map) {
7141                 rc = EBUSY;
7142                 goto done;
7143         }
7144 #endif
7145
7146 #ifdef notyet
7147         rc = -t4_set_filter_mode(sc, fconf);
7148         if (rc == 0)
7149                 sc->filter_mode = fconf;
7150 #else
7151         rc = ENOTSUP;
7152 #endif
7153
7154 done:
7155         end_synchronized_op(sc, LOCK_HELD);
7156         return (rc);
7157 }
7158
7159 static inline uint64_t
7160 get_filter_hits(struct adapter *sc, uint32_t fid)
7161 {
7162         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7163         uint64_t hits;
7164
7165         memwin_info(sc, 0, &mw_base, NULL);
7166         off = position_memwin(sc, 0,
7167             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7168         if (is_t4(sc)) {
7169                 hits = t4_read_reg64(sc, mw_base + off + 16);
7170                 hits = be64toh(hits);
7171         } else {
7172                 hits = t4_read_reg(sc, mw_base + off + 24);
7173                 hits = be32toh(hits);
7174         }
7175
7176         return (hits);
7177 }
7178
7179 static int
7180 get_filter(struct adapter *sc, struct t4_filter *t)
7181 {
7182         int i, rc, nfilters = sc->tids.nftids;
7183         struct filter_entry *f;
7184
7185         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7186             "t4getf");
7187         if (rc)
7188                 return (rc);
7189
7190         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7191             t->idx >= nfilters) {
7192                 t->idx = 0xffffffff;
7193                 goto done;
7194         }
7195
7196         f = &sc->tids.ftid_tab[t->idx];
7197         for (i = t->idx; i < nfilters; i++, f++) {
7198                 if (f->valid) {
7199                         t->idx = i;
7200                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7201                         t->smtidx = f->smtidx;
7202                         if (f->fs.hitcnts)
7203                                 t->hits = get_filter_hits(sc, t->idx);
7204                         else
7205                                 t->hits = UINT64_MAX;
7206                         t->fs = f->fs;
7207
7208                         goto done;
7209                 }
7210         }
7211
7212         t->idx = 0xffffffff;
7213 done:
7214         end_synchronized_op(sc, LOCK_HELD);
7215         return (0);
7216 }
7217
7218 static int
7219 set_filter(struct adapter *sc, struct t4_filter *t)
7220 {
7221         unsigned int nfilters, nports;
7222         struct filter_entry *f;
7223         int i, rc;
7224
7225         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7226         if (rc)
7227                 return (rc);
7228
7229         nfilters = sc->tids.nftids;
7230         nports = sc->params.nports;
7231
7232         if (nfilters == 0) {
7233                 rc = ENOTSUP;
7234                 goto done;
7235         }
7236
7237         if (!(sc->flags & FULL_INIT_DONE)) {
7238                 rc = EAGAIN;
7239                 goto done;
7240         }
7241
7242         if (t->idx >= nfilters) {
7243                 rc = EINVAL;
7244                 goto done;
7245         }
7246
7247         /* Validate against the global filter mode */
7248         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7249             sc->params.tp.vlan_pri_map) {
7250                 rc = E2BIG;
7251                 goto done;
7252         }
7253
7254         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7255                 rc = EINVAL;
7256                 goto done;
7257         }
7258
7259         if (t->fs.val.iport >= nports) {
7260                 rc = EINVAL;
7261                 goto done;
7262         }
7263
7264         /* Can't specify an iq if not steering to it */
7265         if (!t->fs.dirsteer && t->fs.iq) {
7266                 rc = EINVAL;
7267                 goto done;
7268         }
7269
7270         /* IPv6 filter idx must be 4 aligned */
7271         if (t->fs.type == 1 &&
7272             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7273                 rc = EINVAL;
7274                 goto done;
7275         }
7276
7277         if (sc->tids.ftid_tab == NULL) {
7278                 KASSERT(sc->tids.ftids_in_use == 0,
7279                     ("%s: no memory allocated but filters_in_use > 0",
7280                     __func__));
7281
7282                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7283                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7284                 if (sc->tids.ftid_tab == NULL) {
7285                         rc = ENOMEM;
7286                         goto done;
7287                 }
7288                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7289         }
7290
7291         for (i = 0; i < 4; i++) {
7292                 f = &sc->tids.ftid_tab[t->idx + i];
7293
7294                 if (f->pending || f->valid) {
7295                         rc = EBUSY;
7296                         goto done;
7297                 }
7298                 if (f->locked) {
7299                         rc = EPERM;
7300                         goto done;
7301                 }
7302
7303                 if (t->fs.type == 0)
7304                         break;
7305         }
7306
7307         f = &sc->tids.ftid_tab[t->idx];
7308         f->fs = t->fs;
7309
7310         rc = set_filter_wr(sc, t->idx);
7311 done:
7312         end_synchronized_op(sc, 0);
7313
7314         if (rc == 0) {
7315                 mtx_lock(&sc->tids.ftid_lock);
7316                 for (;;) {
7317                         if (f->pending == 0) {
7318                                 rc = f->valid ? 0 : EIO;
7319                                 break;
7320                         }
7321
7322                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7323                             PCATCH, "t4setfw", 0)) {
7324                                 rc = EINPROGRESS;
7325                                 break;
7326                         }
7327                 }
7328                 mtx_unlock(&sc->tids.ftid_lock);
7329         }
7330         return (rc);
7331 }
7332
7333 static int
7334 del_filter(struct adapter *sc, struct t4_filter *t)
7335 {
7336         unsigned int nfilters;
7337         struct filter_entry *f;
7338         int rc;
7339
7340         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7341         if (rc)
7342                 return (rc);
7343
7344         nfilters = sc->tids.nftids;
7345
7346         if (nfilters == 0) {
7347                 rc = ENOTSUP;
7348                 goto done;
7349         }
7350
7351         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7352             t->idx >= nfilters) {
7353                 rc = EINVAL;
7354                 goto done;
7355         }
7356
7357         if (!(sc->flags & FULL_INIT_DONE)) {
7358                 rc = EAGAIN;
7359                 goto done;
7360         }
7361
7362         f = &sc->tids.ftid_tab[t->idx];
7363
7364         if (f->pending) {
7365                 rc = EBUSY;
7366                 goto done;
7367         }
7368         if (f->locked) {
7369                 rc = EPERM;
7370                 goto done;
7371         }
7372
7373         if (f->valid) {
7374                 t->fs = f->fs;  /* extra info for the caller */
7375                 rc = del_filter_wr(sc, t->idx);
7376         }
7377
7378 done:
7379         end_synchronized_op(sc, 0);
7380
7381         if (rc == 0) {
7382                 mtx_lock(&sc->tids.ftid_lock);
7383                 for (;;) {
7384                         if (f->pending == 0) {
7385                                 rc = f->valid ? EIO : 0;
7386                                 break;
7387                         }
7388
7389                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7390                             PCATCH, "t4delfw", 0)) {
7391                                 rc = EINPROGRESS;
7392                                 break;
7393                         }
7394                 }
7395                 mtx_unlock(&sc->tids.ftid_lock);
7396         }
7397
7398         return (rc);
7399 }
7400
7401 static void
7402 clear_filter(struct filter_entry *f)
7403 {
7404         if (f->l2t)
7405                 t4_l2t_release(f->l2t);
7406
7407         bzero(f, sizeof (*f));
7408 }
7409
7410 static int
7411 set_filter_wr(struct adapter *sc, int fidx)
7412 {
7413         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7414         struct fw_filter_wr *fwr;
7415         unsigned int ftid;
7416         struct wrq_cookie cookie;
7417
7418         ASSERT_SYNCHRONIZED_OP(sc);
7419
7420         if (f->fs.newdmac || f->fs.newvlan) {
7421                 /* This filter needs an L2T entry; allocate one. */
7422                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7423                 if (f->l2t == NULL)
7424                         return (EAGAIN);
7425                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7426                     f->fs.dmac)) {
7427                         t4_l2t_release(f->l2t);
7428                         f->l2t = NULL;
7429                         return (ENOMEM);
7430                 }
7431         }
7432
7433         ftid = sc->tids.ftid_base + fidx;
7434
7435         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7436         if (fwr == NULL)
7437                 return (ENOMEM);
7438         bzero(fwr, sizeof(*fwr));
7439
7440         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7441         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7442         fwr->tid_to_iq =
7443             htobe32(V_FW_FILTER_WR_TID(ftid) |
7444                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7445                 V_FW_FILTER_WR_NOREPLY(0) |
7446                 V_FW_FILTER_WR_IQ(f->fs.iq));
7447         fwr->del_filter_to_l2tix =
7448             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7449                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7450                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7451                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7452                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7453                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7454                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7455                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7456                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7457                     f->fs.newvlan == VLAN_REWRITE) |
7458                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7459                     f->fs.newvlan == VLAN_REWRITE) |
7460                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7461                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7462                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7463                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7464         fwr->ethtype = htobe16(f->fs.val.ethtype);
7465         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7466         fwr->frag_to_ovlan_vldm =
7467             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7468                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7469                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7470                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7471                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7472                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7473         fwr->smac_sel = 0;
7474         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7475             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7476         fwr->maci_to_matchtypem =
7477             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7478                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7479                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7480                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7481                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7482                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7483                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7484                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7485         fwr->ptcl = f->fs.val.proto;
7486         fwr->ptclm = f->fs.mask.proto;
7487         fwr->ttyp = f->fs.val.tos;
7488         fwr->ttypm = f->fs.mask.tos;
7489         fwr->ivlan = htobe16(f->fs.val.vlan);
7490         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7491         fwr->ovlan = htobe16(f->fs.val.vnic);
7492         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7493         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7494         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7495         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7496         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7497         fwr->lp = htobe16(f->fs.val.dport);
7498         fwr->lpm = htobe16(f->fs.mask.dport);
7499         fwr->fp = htobe16(f->fs.val.sport);
7500         fwr->fpm = htobe16(f->fs.mask.sport);
7501         if (f->fs.newsmac)
7502                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7503
7504         f->pending = 1;
7505         sc->tids.ftids_in_use++;
7506
7507         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7508         return (0);
7509 }
7510
7511 static int
7512 del_filter_wr(struct adapter *sc, int fidx)
7513 {
7514         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7515         struct fw_filter_wr *fwr;
7516         unsigned int ftid;
7517         struct wrq_cookie cookie;
7518
7519         ftid = sc->tids.ftid_base + fidx;
7520
7521         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7522         if (fwr == NULL)
7523                 return (ENOMEM);
7524         bzero(fwr, sizeof (*fwr));
7525
7526         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7527
7528         f->pending = 1;
7529         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7530         return (0);
7531 }
7532
7533 int
7534 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7535 {
7536         struct adapter *sc = iq->adapter;
7537         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7538         unsigned int idx = GET_TID(rpl);
7539         unsigned int rc;
7540         struct filter_entry *f;
7541
7542         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7543             rss->opcode));
7544
7545         if (is_ftid(sc, idx)) {
7546
7547                 idx -= sc->tids.ftid_base;
7548                 f = &sc->tids.ftid_tab[idx];
7549                 rc = G_COOKIE(rpl->cookie);
7550
7551                 mtx_lock(&sc->tids.ftid_lock);
7552                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7553                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7554                             __func__, idx));
7555                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7556                         f->pending = 0;  /* asynchronous setup completed */
7557                         f->valid = 1;
7558                 } else {
7559                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7560                                 /* Add or delete failed, display an error */
7561                                 log(LOG_ERR,
7562                                     "filter %u setup failed with error %u\n",
7563                                     idx, rc);
7564                         }
7565
7566                         clear_filter(f);
7567                         sc->tids.ftids_in_use--;
7568                 }
7569                 wakeup(&sc->tids.ftid_tab);
7570                 mtx_unlock(&sc->tids.ftid_lock);
7571         }
7572
7573         return (0);
7574 }
7575
7576 static int
7577 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7578 {
7579         int rc;
7580
7581         if (cntxt->cid > M_CTXTQID)
7582                 return (EINVAL);
7583
7584         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7585             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7586                 return (EINVAL);
7587
7588         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7589         if (rc)
7590                 return (rc);
7591
7592         if (sc->flags & FW_OK) {
7593                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7594                     &cntxt->data[0]);
7595                 if (rc == 0)
7596                         goto done;
7597         }
7598
7599         /*
7600          * Read via firmware failed or wasn't even attempted.  Read directly via
7601          * the backdoor.
7602          */
7603         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7604 done:
7605         end_synchronized_op(sc, 0);
7606         return (rc);
7607 }
7608
7609 static int
7610 load_fw(struct adapter *sc, struct t4_data *fw)
7611 {
7612         int rc;
7613         uint8_t *fw_data;
7614
7615         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7616         if (rc)
7617                 return (rc);
7618
7619         if (sc->flags & FULL_INIT_DONE) {
7620                 rc = EBUSY;
7621                 goto done;
7622         }
7623
7624         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7625         if (fw_data == NULL) {
7626                 rc = ENOMEM;
7627                 goto done;
7628         }
7629
7630         rc = copyin(fw->data, fw_data, fw->len);
7631         if (rc == 0)
7632                 rc = -t4_load_fw(sc, fw_data, fw->len);
7633
7634         free(fw_data, M_CXGBE);
7635 done:
7636         end_synchronized_op(sc, 0);
7637         return (rc);
7638 }
7639
7640 static int
7641 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7642 {
7643         uint32_t addr, off, remaining, i, n;
7644         uint32_t *buf, *b;
7645         uint32_t mw_base, mw_aperture;
7646         int rc;
7647         uint8_t *dst;
7648
7649         rc = validate_mem_range(sc, mr->addr, mr->len);
7650         if (rc != 0)
7651                 return (rc);
7652
7653         memwin_info(sc, win, &mw_base, &mw_aperture);
7654         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7655         addr = mr->addr;
7656         remaining = mr->len;
7657         dst = (void *)mr->data;
7658
7659         while (remaining) {
7660                 off = position_memwin(sc, win, addr);
7661
7662                 /* number of bytes that we'll copy in the inner loop */
7663                 n = min(remaining, mw_aperture - off);
7664                 for (i = 0; i < n; i += 4)
7665                         *b++ = t4_read_reg(sc, mw_base + off + i);
7666
7667                 rc = copyout(buf, dst, n);
7668                 if (rc != 0)
7669                         break;
7670
7671                 b = buf;
7672                 dst += n;
7673                 remaining -= n;
7674                 addr += n;
7675         }
7676
7677         free(buf, M_CXGBE);
7678         return (rc);
7679 }
7680
7681 static int
7682 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7683 {
7684         int rc;
7685
7686         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7687                 return (EINVAL);
7688
7689         if (i2cd->len > sizeof(i2cd->data))
7690                 return (EFBIG);
7691
7692         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7693         if (rc)
7694                 return (rc);
7695         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7696             i2cd->offset, i2cd->len, &i2cd->data[0]);
7697         end_synchronized_op(sc, 0);
7698
7699         return (rc);
7700 }
7701
7702 static int
7703 in_range(int val, int lo, int hi)
7704 {
7705
7706         return (val < 0 || (val <= hi && val >= lo));
7707 }
7708
7709 static int
7710 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7711 {
7712         int fw_subcmd, fw_type, rc;
7713
7714         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7715         if (rc)
7716                 return (rc);
7717
7718         if (!(sc->flags & FULL_INIT_DONE)) {
7719                 rc = EAGAIN;
7720                 goto done;
7721         }
7722
7723         /*
7724          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7725          * sub-command and type are in common locations.)
7726          */
7727         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7728                 fw_subcmd = FW_SCHED_SC_CONFIG;
7729         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7730                 fw_subcmd = FW_SCHED_SC_PARAMS;
7731         else {
7732                 rc = EINVAL;
7733                 goto done;
7734         }
7735         if (p->type == SCHED_CLASS_TYPE_PACKET)
7736                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7737         else {
7738                 rc = EINVAL;
7739                 goto done;
7740         }
7741
7742         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7743                 /* Vet our parameters ..*/
7744                 if (p->u.config.minmax < 0) {
7745                         rc = EINVAL;
7746                         goto done;
7747                 }
7748
7749                 /* And pass the request to the firmware ...*/
7750                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7751                 goto done;
7752         }
7753
7754         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7755                 int fw_level;
7756                 int fw_mode;
7757                 int fw_rateunit;
7758                 int fw_ratemode;
7759
7760                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7761                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7762                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7763                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7764                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7765                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7766                 else {
7767                         rc = EINVAL;
7768                         goto done;
7769                 }
7770
7771                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7772                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7773                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7774                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7775                 else {
7776                         rc = EINVAL;
7777                         goto done;
7778                 }
7779
7780                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7781                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7782                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7783                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7784                 else {
7785                         rc = EINVAL;
7786                         goto done;
7787                 }
7788
7789                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7790                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7791                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7792                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7793                 else {
7794                         rc = EINVAL;
7795                         goto done;
7796                 }
7797
7798                 /* Vet our parameters ... */
7799                 if (!in_range(p->u.params.channel, 0, 3) ||
7800                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7801                     !in_range(p->u.params.minrate, 0, 10000000) ||
7802                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7803                     !in_range(p->u.params.weight, 0, 100)) {
7804                         rc = ERANGE;
7805                         goto done;
7806                 }
7807
7808                 /*
7809                  * Translate any unset parameters into the firmware's
7810                  * nomenclature and/or fail the call if the parameters
7811                  * are required ...
7812                  */
7813                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7814                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7815                         rc = EINVAL;
7816                         goto done;
7817                 }
7818                 if (p->u.params.minrate < 0)
7819                         p->u.params.minrate = 0;
7820                 if (p->u.params.maxrate < 0) {
7821                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7822                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7823                                 rc = EINVAL;
7824                                 goto done;
7825                         } else
7826                                 p->u.params.maxrate = 0;
7827                 }
7828                 if (p->u.params.weight < 0) {
7829                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7830                                 rc = EINVAL;
7831                                 goto done;
7832                         } else
7833                                 p->u.params.weight = 0;
7834                 }
7835                 if (p->u.params.pktsize < 0) {
7836                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7837                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7838                                 rc = EINVAL;
7839                                 goto done;
7840                         } else
7841                                 p->u.params.pktsize = 0;
7842                 }
7843
7844                 /* See what the firmware thinks of the request ... */
7845                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7846                     fw_rateunit, fw_ratemode, p->u.params.channel,
7847                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7848                     p->u.params.weight, p->u.params.pktsize, 1);
7849                 goto done;
7850         }
7851
7852         rc = EINVAL;
7853 done:
7854         end_synchronized_op(sc, 0);
7855         return (rc);
7856 }
7857
7858 static int
7859 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7860 {
7861         struct port_info *pi = NULL;
7862         struct sge_txq *txq;
7863         uint32_t fw_mnem, fw_queue, fw_class;
7864         int i, rc;
7865
7866         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7867         if (rc)
7868                 return (rc);
7869
7870         if (!(sc->flags & FULL_INIT_DONE)) {
7871                 rc = EAGAIN;
7872                 goto done;
7873         }
7874
7875         if (p->port >= sc->params.nports) {
7876                 rc = EINVAL;
7877                 goto done;
7878         }
7879
7880         pi = sc->port[p->port];
7881         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7882                 rc = EINVAL;
7883                 goto done;
7884         }
7885
7886         /*
7887          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7888          * Scheduling Class in this case).
7889          */
7890         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7891             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7892         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7893
7894         /*
7895          * If op.queue is non-negative, then we're only changing the scheduling
7896          * on a single specified TX queue.
7897          */
7898         if (p->queue >= 0) {
7899                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7900                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7901                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7902                     &fw_class);
7903                 goto done;
7904         }
7905
7906         /*
7907          * Change the scheduling on all the TX queues for the
7908          * interface.
7909          */
7910         for_each_txq(pi, i, txq) {
7911                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7912                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7913                     &fw_class);
7914                 if (rc)
7915                         goto done;
7916         }
7917
7918         rc = 0;
7919 done:
7920         end_synchronized_op(sc, 0);
7921         return (rc);
7922 }
7923
7924 int
7925 t4_os_find_pci_capability(struct adapter *sc, int cap)
7926 {
7927         int i;
7928
7929         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7930 }
7931
7932 int
7933 t4_os_pci_save_state(struct adapter *sc)
7934 {
7935         device_t dev;
7936         struct pci_devinfo *dinfo;
7937
7938         dev = sc->dev;
7939         dinfo = device_get_ivars(dev);
7940
7941         pci_cfg_save(dev, dinfo, 0);
7942         return (0);
7943 }
7944
7945 int
7946 t4_os_pci_restore_state(struct adapter *sc)
7947 {
7948         device_t dev;
7949         struct pci_devinfo *dinfo;
7950
7951         dev = sc->dev;
7952         dinfo = device_get_ivars(dev);
7953
7954         pci_cfg_restore(dev, dinfo);
7955         return (0);
7956 }
7957
7958 void
7959 t4_os_portmod_changed(const struct adapter *sc, int idx)
7960 {
7961         struct port_info *pi = sc->port[idx];
7962         static const char *mod_str[] = {
7963                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7964         };
7965
7966         build_medialist(pi, &pi->media);
7967 #ifdef DEV_NETMAP
7968         build_medialist(pi, &pi->nm_media);
7969 #endif
7970
7971         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7972                 if_printf(pi->ifp, "transceiver unplugged.\n");
7973         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7974                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7975         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7976                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7977         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7978                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7979                     mod_str[pi->mod_type]);
7980         } else {
7981                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7982                     pi->mod_type);
7983         }
7984 }
7985
7986 void
7987 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7988 {
7989         struct port_info *pi = sc->port[idx];
7990         struct ifnet *ifp = pi->ifp;
7991
7992         if (link_stat) {
7993                 pi->linkdnrc = -1;
7994                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7995                 if_link_state_change(ifp, LINK_STATE_UP);
7996         } else {
7997                 if (reason >= 0)
7998                         pi->linkdnrc = reason;
7999                 if_link_state_change(ifp, LINK_STATE_DOWN);
8000         }
8001 }
8002
8003 void
8004 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8005 {
8006         struct adapter *sc;
8007
8008         sx_slock(&t4_list_lock);
8009         SLIST_FOREACH(sc, &t4_list, link) {
8010                 /*
8011                  * func should not make any assumptions about what state sc is
8012                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8013                  */
8014                 func(sc, arg);
8015         }
8016         sx_sunlock(&t4_list_lock);
8017 }
8018
8019 static int
8020 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8021 {
8022        return (0);
8023 }
8024
8025 static int
8026 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8027 {
8028        return (0);
8029 }
8030
8031 static int
8032 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8033     struct thread *td)
8034 {
8035         int rc;
8036         struct adapter *sc = dev->si_drv1;
8037
8038         rc = priv_check(td, PRIV_DRIVER);
8039         if (rc != 0)
8040                 return (rc);
8041
8042         switch (cmd) {
8043         case CHELSIO_T4_GETREG: {
8044                 struct t4_reg *edata = (struct t4_reg *)data;
8045
8046                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8047                         return (EFAULT);
8048
8049                 if (edata->size == 4)
8050                         edata->val = t4_read_reg(sc, edata->addr);
8051                 else if (edata->size == 8)
8052                         edata->val = t4_read_reg64(sc, edata->addr);
8053                 else
8054                         return (EINVAL);
8055
8056                 break;
8057         }
8058         case CHELSIO_T4_SETREG: {
8059                 struct t4_reg *edata = (struct t4_reg *)data;
8060
8061                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8062                         return (EFAULT);
8063
8064                 if (edata->size == 4) {
8065                         if (edata->val & 0xffffffff00000000)
8066                                 return (EINVAL);
8067                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8068                 } else if (edata->size == 8)
8069                         t4_write_reg64(sc, edata->addr, edata->val);
8070                 else
8071                         return (EINVAL);
8072                 break;
8073         }
8074         case CHELSIO_T4_REGDUMP: {
8075                 struct t4_regdump *regs = (struct t4_regdump *)data;
8076                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8077                 uint8_t *buf;
8078
8079                 if (regs->len < reglen) {
8080                         regs->len = reglen; /* hint to the caller */
8081                         return (ENOBUFS);
8082                 }
8083
8084                 regs->len = reglen;
8085                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8086                 t4_get_regs(sc, regs, buf);
8087                 rc = copyout(buf, regs->data, reglen);
8088                 free(buf, M_CXGBE);
8089                 break;
8090         }
8091         case CHELSIO_T4_GET_FILTER_MODE:
8092                 rc = get_filter_mode(sc, (uint32_t *)data);
8093                 break;
8094         case CHELSIO_T4_SET_FILTER_MODE:
8095                 rc = set_filter_mode(sc, *(uint32_t *)data);
8096                 break;
8097         case CHELSIO_T4_GET_FILTER:
8098                 rc = get_filter(sc, (struct t4_filter *)data);
8099                 break;
8100         case CHELSIO_T4_SET_FILTER:
8101                 rc = set_filter(sc, (struct t4_filter *)data);
8102                 break;
8103         case CHELSIO_T4_DEL_FILTER:
8104                 rc = del_filter(sc, (struct t4_filter *)data);
8105                 break;
8106         case CHELSIO_T4_GET_SGE_CONTEXT:
8107                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8108                 break;
8109         case CHELSIO_T4_LOAD_FW:
8110                 rc = load_fw(sc, (struct t4_data *)data);
8111                 break;
8112         case CHELSIO_T4_GET_MEM:
8113                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8114                 break;
8115         case CHELSIO_T4_GET_I2C:
8116                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8117                 break;
8118         case CHELSIO_T4_CLEAR_STATS: {
8119                 int i;
8120                 u_int port_id = *(uint32_t *)data;
8121                 struct port_info *pi;
8122
8123                 if (port_id >= sc->params.nports)
8124                         return (EINVAL);
8125                 pi = sc->port[port_id];
8126
8127                 /* MAC stats */
8128                 t4_clr_port_stats(sc, pi->tx_chan);
8129                 pi->tx_parse_error = 0;
8130
8131                 if (pi->flags & PORT_INIT_DONE) {
8132                         struct sge_rxq *rxq;
8133                         struct sge_txq *txq;
8134                         struct sge_wrq *wrq;
8135
8136                         for_each_rxq(pi, i, rxq) {
8137 #if defined(INET) || defined(INET6)
8138                                 rxq->lro.lro_queued = 0;
8139                                 rxq->lro.lro_flushed = 0;
8140 #endif
8141                                 rxq->rxcsum = 0;
8142                                 rxq->vlan_extraction = 0;
8143                         }
8144
8145                         for_each_txq(pi, i, txq) {
8146                                 txq->txcsum = 0;
8147                                 txq->tso_wrs = 0;
8148                                 txq->vlan_insertion = 0;
8149                                 txq->imm_wrs = 0;
8150                                 txq->sgl_wrs = 0;
8151                                 txq->txpkt_wrs = 0;
8152                                 txq->txpkts0_wrs = 0;
8153                                 txq->txpkts1_wrs = 0;
8154                                 txq->txpkts0_pkts = 0;
8155                                 txq->txpkts1_pkts = 0;
8156                                 mp_ring_reset_stats(txq->r);
8157                         }
8158
8159 #ifdef TCP_OFFLOAD
8160                         /* nothing to clear for each ofld_rxq */
8161
8162                         for_each_ofld_txq(pi, i, wrq) {
8163                                 wrq->tx_wrs_direct = 0;
8164                                 wrq->tx_wrs_copied = 0;
8165                         }
8166 #endif
8167                         wrq = &sc->sge.ctrlq[pi->port_id];
8168                         wrq->tx_wrs_direct = 0;
8169                         wrq->tx_wrs_copied = 0;
8170                 }
8171                 break;
8172         }
8173         case CHELSIO_T4_SCHED_CLASS:
8174                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8175                 break;
8176         case CHELSIO_T4_SCHED_QUEUE:
8177                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8178                 break;
8179         case CHELSIO_T4_GET_TRACER:
8180                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8181                 break;
8182         case CHELSIO_T4_SET_TRACER:
8183                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8184                 break;
8185         default:
8186                 rc = EINVAL;
8187         }
8188
8189         return (rc);
8190 }
8191
8192 #ifdef TCP_OFFLOAD
8193 void
8194 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8195     const unsigned int *pgsz_order)
8196 {
8197         struct port_info *pi = ifp->if_softc;
8198         struct adapter *sc = pi->adapter;
8199
8200         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8201         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8202                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8203                 V_HPZ3(pgsz_order[3]));
8204 }
8205
8206 static int
8207 toe_capability(struct port_info *pi, int enable)
8208 {
8209         int rc;
8210         struct adapter *sc = pi->adapter;
8211
8212         ASSERT_SYNCHRONIZED_OP(sc);
8213
8214         if (!is_offload(sc))
8215                 return (ENODEV);
8216
8217         if (enable) {
8218                 if (!(sc->flags & FULL_INIT_DONE)) {
8219                         rc = cxgbe_init_synchronized(pi);
8220                         if (rc)
8221                                 return (rc);
8222                 }
8223
8224                 if (isset(&sc->offload_map, pi->port_id))
8225                         return (0);
8226
8227                 if (!(sc->flags & TOM_INIT_DONE)) {
8228                         rc = t4_activate_uld(sc, ULD_TOM);
8229                         if (rc == EAGAIN) {
8230                                 log(LOG_WARNING,
8231                                     "You must kldload t4_tom.ko before trying "
8232                                     "to enable TOE on a cxgbe interface.\n");
8233                         }
8234                         if (rc != 0)
8235                                 return (rc);
8236                         KASSERT(sc->tom_softc != NULL,
8237                             ("%s: TOM activated but softc NULL", __func__));
8238                         KASSERT(sc->flags & TOM_INIT_DONE,
8239                             ("%s: TOM activated but flag not set", __func__));
8240                 }
8241
8242                 setbit(&sc->offload_map, pi->port_id);
8243         } else {
8244                 if (!isset(&sc->offload_map, pi->port_id))
8245                         return (0);
8246
8247                 KASSERT(sc->flags & TOM_INIT_DONE,
8248                     ("%s: TOM never initialized?", __func__));
8249                 clrbit(&sc->offload_map, pi->port_id);
8250         }
8251
8252         return (0);
8253 }
8254
8255 /*
8256  * Add an upper layer driver to the global list.
8257  */
8258 int
8259 t4_register_uld(struct uld_info *ui)
8260 {
8261         int rc = 0;
8262         struct uld_info *u;
8263
8264         sx_xlock(&t4_uld_list_lock);
8265         SLIST_FOREACH(u, &t4_uld_list, link) {
8266             if (u->uld_id == ui->uld_id) {
8267                     rc = EEXIST;
8268                     goto done;
8269             }
8270         }
8271
8272         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8273         ui->refcount = 0;
8274 done:
8275         sx_xunlock(&t4_uld_list_lock);
8276         return (rc);
8277 }
8278
8279 int
8280 t4_unregister_uld(struct uld_info *ui)
8281 {
8282         int rc = EINVAL;
8283         struct uld_info *u;
8284
8285         sx_xlock(&t4_uld_list_lock);
8286
8287         SLIST_FOREACH(u, &t4_uld_list, link) {
8288             if (u == ui) {
8289                     if (ui->refcount > 0) {
8290                             rc = EBUSY;
8291                             goto done;
8292                     }
8293
8294                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8295                     rc = 0;
8296                     goto done;
8297             }
8298         }
8299 done:
8300         sx_xunlock(&t4_uld_list_lock);
8301         return (rc);
8302 }
8303
8304 int
8305 t4_activate_uld(struct adapter *sc, int id)
8306 {
8307         int rc = EAGAIN;
8308         struct uld_info *ui;
8309
8310         ASSERT_SYNCHRONIZED_OP(sc);
8311
8312         sx_slock(&t4_uld_list_lock);
8313
8314         SLIST_FOREACH(ui, &t4_uld_list, link) {
8315                 if (ui->uld_id == id) {
8316                         if (!(sc->flags & FULL_INIT_DONE)) {
8317                                 rc = adapter_full_init(sc);
8318                                 if (rc != 0)
8319                                         goto done;
8320                         }
8321
8322                         rc = ui->activate(sc);
8323                         if (rc == 0)
8324                                 ui->refcount++;
8325                         goto done;
8326                 }
8327         }
8328 done:
8329         sx_sunlock(&t4_uld_list_lock);
8330
8331         return (rc);
8332 }
8333
8334 int
8335 t4_deactivate_uld(struct adapter *sc, int id)
8336 {
8337         int rc = EINVAL;
8338         struct uld_info *ui;
8339
8340         ASSERT_SYNCHRONIZED_OP(sc);
8341
8342         sx_slock(&t4_uld_list_lock);
8343
8344         SLIST_FOREACH(ui, &t4_uld_list, link) {
8345                 if (ui->uld_id == id) {
8346                         rc = ui->deactivate(sc);
8347                         if (rc == 0)
8348                                 ui->refcount--;
8349                         goto done;
8350                 }
8351         }
8352 done:
8353         sx_sunlock(&t4_uld_list_lock);
8354
8355         return (rc);
8356 }
8357 #endif
8358
8359 /*
8360  * Come up with reasonable defaults for some of the tunables, provided they're
8361  * not set by the user (in which case we'll use the values as is).
8362  */
8363 static void
8364 tweak_tunables(void)
8365 {
8366         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8367
8368         if (t4_ntxq10g < 1)
8369                 t4_ntxq10g = min(nc, NTXQ_10G);
8370
8371         if (t4_ntxq1g < 1)
8372                 t4_ntxq1g = min(nc, NTXQ_1G);
8373
8374         if (t4_nrxq10g < 1)
8375                 t4_nrxq10g = min(nc, NRXQ_10G);
8376
8377         if (t4_nrxq1g < 1)
8378                 t4_nrxq1g = min(nc, NRXQ_1G);
8379
8380 #ifdef TCP_OFFLOAD
8381         if (t4_nofldtxq10g < 1)
8382                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8383
8384         if (t4_nofldtxq1g < 1)
8385                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8386
8387         if (t4_nofldrxq10g < 1)
8388                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8389
8390         if (t4_nofldrxq1g < 1)
8391                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8392
8393         if (t4_toecaps_allowed == -1)
8394                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8395 #else
8396         if (t4_toecaps_allowed == -1)
8397                 t4_toecaps_allowed = 0;
8398 #endif
8399
8400 #ifdef DEV_NETMAP
8401         if (t4_nnmtxq10g < 1)
8402                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8403
8404         if (t4_nnmtxq1g < 1)
8405                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8406
8407         if (t4_nnmrxq10g < 1)
8408                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8409
8410         if (t4_nnmrxq1g < 1)
8411                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8412 #endif
8413
8414         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8415                 t4_tmr_idx_10g = TMR_IDX_10G;
8416
8417         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8418                 t4_pktc_idx_10g = PKTC_IDX_10G;
8419
8420         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8421                 t4_tmr_idx_1g = TMR_IDX_1G;
8422
8423         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8424                 t4_pktc_idx_1g = PKTC_IDX_1G;
8425
8426         if (t4_qsize_txq < 128)
8427                 t4_qsize_txq = 128;
8428
8429         if (t4_qsize_rxq < 128)
8430                 t4_qsize_rxq = 128;
8431         while (t4_qsize_rxq & 7)
8432                 t4_qsize_rxq++;
8433
8434         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8435 }
8436
8437 static struct sx mlu;   /* mod load unload */
8438 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8439
8440 static int
8441 mod_event(module_t mod, int cmd, void *arg)
8442 {
8443         int rc = 0;
8444         static int loaded = 0;
8445
8446         switch (cmd) {
8447         case MOD_LOAD:
8448                 sx_xlock(&mlu);
8449                 if (loaded++ == 0) {
8450                         t4_sge_modload();
8451                         sx_init(&t4_list_lock, "T4/T5 adapters");
8452                         SLIST_INIT(&t4_list);
8453 #ifdef TCP_OFFLOAD
8454                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8455                         SLIST_INIT(&t4_uld_list);
8456 #endif
8457                         t4_tracer_modload();
8458                         tweak_tunables();
8459                 }
8460                 sx_xunlock(&mlu);
8461                 break;
8462
8463         case MOD_UNLOAD:
8464                 sx_xlock(&mlu);
8465                 if (--loaded == 0) {
8466                         int tries;
8467
8468                         sx_slock(&t4_list_lock);
8469                         if (!SLIST_EMPTY(&t4_list)) {
8470                                 rc = EBUSY;
8471                                 sx_sunlock(&t4_list_lock);
8472                                 goto done_unload;
8473                         }
8474 #ifdef TCP_OFFLOAD
8475                         sx_slock(&t4_uld_list_lock);
8476                         if (!SLIST_EMPTY(&t4_uld_list)) {
8477                                 rc = EBUSY;
8478                                 sx_sunlock(&t4_uld_list_lock);
8479                                 sx_sunlock(&t4_list_lock);
8480                                 goto done_unload;
8481                         }
8482 #endif
8483                         tries = 0;
8484                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8485                                 uprintf("%ju clusters with custom free routine "
8486                                     "still is use.\n", t4_sge_extfree_refs());
8487                                 pause("t4unload", 2 * hz);
8488                         }
8489 #ifdef TCP_OFFLOAD
8490                         sx_sunlock(&t4_uld_list_lock);
8491 #endif
8492                         sx_sunlock(&t4_list_lock);
8493
8494                         if (t4_sge_extfree_refs() == 0) {
8495                                 t4_tracer_modunload();
8496 #ifdef TCP_OFFLOAD
8497                                 sx_destroy(&t4_uld_list_lock);
8498 #endif
8499                                 sx_destroy(&t4_list_lock);
8500                                 t4_sge_modunload();
8501                                 loaded = 0;
8502                         } else {
8503                                 rc = EBUSY;
8504                                 loaded++;       /* undo earlier decrement */
8505                         }
8506                 }
8507 done_unload:
8508                 sx_xunlock(&mlu);
8509                 break;
8510         }
8511
8512         return (rc);
8513 }
8514
8515 static devclass_t t4_devclass, t5_devclass;
8516 static devclass_t cxgbe_devclass, cxl_devclass;
8517
8518 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8519 MODULE_VERSION(t4nex, 1);
8520 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8521
8522 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8523 MODULE_VERSION(t5nex, 1);
8524 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8525
8526 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8527 MODULE_VERSION(cxgbe, 1);
8528
8529 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8530 MODULE_VERSION(cxl, 1);