sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491 #ifdef notyet
 492         {0x5404,  "Chelsio T520-BCH"},
 493         {0x5405,  "Chelsio T540-BCH"},
 494         {0x5406,  "Chelsio T540-CH"},
 495         {0x5408,  "Chelsio T520-CX"},
 496         {0x540b,  "Chelsio B520-SR"},
 497         {0x540c,  "Chelsio B504-BT"},
 498         {0x540f,  "Chelsio Amsterdam"},
 499         {0x5413,  "Chelsio T580-CHR"},
 500 #endif
 501 };
 502
 503 #ifdef TCP_OFFLOAD
 504 /*
 505  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 506  * exactly the same for both rxq and ofld_rxq.
 507  */
 508 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 509 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 510 #endif
 511
 512 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 513 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 514 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 515
 516 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 517
 518 static int
 519 t4_probe(device_t dev)
 520 {
 521         int i;
 522         uint16_t v = pci_get_vendor(dev);
 523         uint16_t d = pci_get_device(dev);
 524         uint8_t f = pci_get_function(dev);
 525
 526         if (v != PCI_VENDOR_ID_CHELSIO)
 527                 return (ENXIO);
 528
 529         /* Attach only to PF0 of the FPGA */
 530         if (d == 0xa000 && f != 0)
 531                 return (ENXIO);
 532
 533         for (i = 0; i < nitems(t4_pciids); i++) {
 534                 if (d == t4_pciids[i].device) {
 535                         device_set_desc(dev, t4_pciids[i].desc);
 536                         return (BUS_PROBE_DEFAULT);
 537                 }
 538         }
 539
 540         return (ENXIO);
 541 }
 542
 543 static int
 544 t5_probe(device_t dev)
 545 {
 546         int i;
 547         uint16_t v = pci_get_vendor(dev);
 548         uint16_t d = pci_get_device(dev);
 549         uint8_t f = pci_get_function(dev);
 550
 551         if (v != PCI_VENDOR_ID_CHELSIO)
 552                 return (ENXIO);
 553
 554         /* Attach only to PF0 of the FPGA */
 555         if (d == 0xb000 && f != 0)
 556                 return (ENXIO);
 557
 558         for (i = 0; i < nitems(t5_pciids); i++) {
 559                 if (d == t5_pciids[i].device) {
 560                         device_set_desc(dev, t5_pciids[i].desc);
 561                         return (BUS_PROBE_DEFAULT);
 562                 }
 563         }
 564
 565         return (ENXIO);
 566 }
 567
 568 static int
 569 t4_attach(device_t dev)
 570 {
 571         struct adapter *sc;
 572         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 573         struct intrs_and_queues iaq;
 574         struct sge *s;
 575 #ifdef TCP_OFFLOAD
 576         int ofld_rqidx, ofld_tqidx;
 577 #endif
 578 #ifdef DEV_NETMAP
 579         int nm_rqidx, nm_tqidx;
 580 #endif
 581         const char *pcie_ts;
 582
 583         sc = device_get_softc(dev);
 584         sc->dev = dev;
 585
 586         pci_enable_busmaster(dev);
 587         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 588                 uint32_t v;
 589
 590                 pci_set_max_read_req(dev, 4096);
 591                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 592                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 593                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 594         }
 595
 596         sc->traceq = -1;
 597         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 598         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 599             device_get_nameunit(dev));
 600
 601         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 602             device_get_nameunit(dev));
 603         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 604         sx_xlock(&t4_list_lock);
 605         SLIST_INSERT_HEAD(&t4_list, sc, link);
 606         sx_xunlock(&t4_list_lock);
 607
 608         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 609         TAILQ_INIT(&sc->sfl);
 610         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 611
 612         rc = map_bars_0_and_4(sc);
 613         if (rc != 0)
 614                 goto done; /* error message displayed already */
 615
 616         /*
 617          * This is the real PF# to which we're attaching.  Works from within PCI
 618          * passthrough environments too, where pci_get_function() could return a
 619          * different PF# depending on the passthrough configuration.  We need to
 620          * use the real PF# in all our communication with the firmware.
 621          */
 622         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 623         sc->mbox = sc->pf;
 624
 625         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 626         sc->an_handler = an_not_handled;
 627         for (i = 0; i < nitems(sc->cpl_handler); i++)
 628                 sc->cpl_handler[i] = cpl_not_handled;
 629         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 630                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 631         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 632         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 633         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 634         t4_init_sge_cpl_handlers(sc);
 635
 636         /* Prepare the adapter for operation */
 637         rc = -t4_prep_adapter(sc);
 638         if (rc != 0) {
 639                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 640                 goto done;
 641         }
 642
 643         /*
 644          * Do this really early, with the memory windows set up even before the
 645          * character device.  The userland tool's register i/o and mem read
 646          * will work even in "recovery mode".
 647          */
 648         setup_memwin(sc);
 649         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 650             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 651             device_get_nameunit(dev));
 652         if (sc->cdev == NULL)
 653                 device_printf(dev, "failed to create nexus char device.\n");
 654         else
 655                 sc->cdev->si_drv1 = sc;
 656
 657         /* Go no further if recovery mode has been requested. */
 658         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 659                 device_printf(dev, "recovery mode.\n");
 660                 goto done;
 661         }
 662
 663         /* Prepare the firmware for operation */
 664         rc = prep_firmware(sc);
 665         if (rc != 0)
 666                 goto done; /* error message displayed already */
 667
 668         rc = get_params__post_init(sc);
 669         if (rc != 0)
 670                 goto done; /* error message displayed already */
 671
 672         rc = set_params__post_init(sc);
 673         if (rc != 0)
 674                 goto done; /* error message displayed already */
 675
 676         rc = map_bar_2(sc);
 677         if (rc != 0)
 678                 goto done; /* error message displayed already */
 679
 680         rc = t4_create_dma_tag(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         /*
 685          * First pass over all the ports - allocate VIs and initialize some
 686          * basic parameters like mac address, port type, etc.  We also figure
 687          * out whether a port is 10G or 1G and use that information when
 688          * calculating how many interrupts to attempt to allocate.
 689          */
 690         n10g = n1g = 0;
 691         for_each_port(sc, i) {
 692                 struct port_info *pi;
 693
 694                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 695                 sc->port[i] = pi;
 696
 697                 /* These must be set before t4_port_init */
 698                 pi->adapter = sc;
 699                 pi->port_id = i;
 700
 701                 /* Allocate the vi and initialize parameters like mac addr */
 702                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 703                 if (rc != 0) {
 704                         device_printf(dev, "unable to initialize port %d: %d\n",
 705                             i, rc);
 706                         free(pi, M_CXGBE);
 707                         sc->port[i] = NULL;
 708                         goto done;
 709                 }
 710
 711                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 712                 pi->link_cfg.requested_fc |= t4_pause_settings;
 713                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 714                 pi->link_cfg.fc |= t4_pause_settings;
 715
 716                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 717                 if (rc != 0) {
 718                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 719                         free(pi, M_CXGBE);
 720                         sc->port[i] = NULL;
 721                         goto done;
 722                 }
 723
 724                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 725                     device_get_nameunit(dev), i);
 726                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 727                 sc->chan_map[pi->tx_chan] = i;
 728
 729                 if (is_10G_port(pi) || is_40G_port(pi)) {
 730                         n10g++;
 731                         pi->tmr_idx = t4_tmr_idx_10g;
 732                         pi->pktc_idx = t4_pktc_idx_10g;
 733                 } else {
 734                         n1g++;
 735                         pi->tmr_idx = t4_tmr_idx_1g;
 736                         pi->pktc_idx = t4_pktc_idx_1g;
 737                 }
 738
 739                 pi->xact_addr_filt = -1;
 740                 pi->linkdnrc = -1;
 741
 742                 pi->qsize_rxq = t4_qsize_rxq;
 743                 pi->qsize_txq = t4_qsize_txq;
 744
 745                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 746                 if (pi->dev == NULL) {
 747                         device_printf(dev,
 748                             "failed to add device for port %d.\n", i);
 749                         rc = ENXIO;
 750                         goto done;
 751                 }
 752                 device_set_softc(pi->dev, pi);
 753         }
 754
 755         /*
 756          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 757          */
 758         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 759         if (rc != 0)
 760                 goto done; /* error message displayed already */
 761
 762         sc->intr_type = iaq.intr_type;
 763         sc->intr_count = iaq.nirq;
 764
 765         s = &sc->sge;
 766         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 767         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 768         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 769         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 770         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 771 #ifdef TCP_OFFLOAD
 772         if (is_offload(sc)) {
 773                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 774                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 775                 s->neq += s->nofldtxq + s->nofldrxq;
 776                 s->niq += s->nofldrxq;
 777
 778                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 779                     M_CXGBE, M_ZERO | M_WAITOK);
 780                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 781                     M_CXGBE, M_ZERO | M_WAITOK);
 782         }
 783 #endif
 784 #ifdef DEV_NETMAP
 785         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 786         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 787         s->neq += s->nnmtxq + s->nnmrxq;
 788         s->niq += s->nnmrxq;
 789
 790         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 791             M_CXGBE, M_ZERO | M_WAITOK);
 792         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 793             M_CXGBE, M_ZERO | M_WAITOK);
 794 #endif
 795
 796         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 797             M_ZERO | M_WAITOK);
 798         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 799             M_ZERO | M_WAITOK);
 800         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 801             M_ZERO | M_WAITOK);
 802         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 803             M_ZERO | M_WAITOK);
 804         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806
 807         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 808             M_ZERO | M_WAITOK);
 809
 810         t4_init_l2t(sc, M_WAITOK);
 811
 812         /*
 813          * Second pass over the ports.  This time we know the number of rx and
 814          * tx queues that each port should get.
 815          */
 816         rqidx = tqidx = 0;
 817 #ifdef TCP_OFFLOAD
 818         ofld_rqidx = ofld_tqidx = 0;
 819 #endif
 820 #ifdef DEV_NETMAP
 821         nm_rqidx = nm_tqidx = 0;
 822 #endif
 823         for_each_port(sc, i) {
 824                 struct port_info *pi = sc->port[i];
 825
 826                 if (pi == NULL)
 827                         continue;
 828
 829                 pi->first_rxq = rqidx;
 830                 pi->first_txq = tqidx;
 831                 if (is_10G_port(pi) || is_40G_port(pi)) {
 832                         pi->flags |= iaq.intr_flags_10g;
 833                         pi->nrxq = iaq.nrxq10g;
 834                         pi->ntxq = iaq.ntxq10g;
 835                 } else {
 836                         pi->flags |= iaq.intr_flags_1g;
 837                         pi->nrxq = iaq.nrxq1g;
 838                         pi->ntxq = iaq.ntxq1g;
 839                 }
 840
 841                 if (pi->ntxq > 1)
 842                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 843                 else
 844                         pi->rsrv_noflowq = 0;
 845
 846                 rqidx += pi->nrxq;
 847                 tqidx += pi->ntxq;
 848 #ifdef TCP_OFFLOAD
 849                 if (is_offload(sc)) {
 850                         pi->first_ofld_rxq = ofld_rqidx;
 851                         pi->first_ofld_txq = ofld_tqidx;
 852                         if (is_10G_port(pi) || is_40G_port(pi)) {
 853                                 pi->nofldrxq = iaq.nofldrxq10g;
 854                                 pi->nofldtxq = iaq.nofldtxq10g;
 855                         } else {
 856                                 pi->nofldrxq = iaq.nofldrxq1g;
 857                                 pi->nofldtxq = iaq.nofldtxq1g;
 858                         }
 859                         ofld_rqidx += pi->nofldrxq;
 860                         ofld_tqidx += pi->nofldtxq;
 861                 }
 862 #endif
 863 #ifdef DEV_NETMAP
 864                 pi->first_nm_rxq = nm_rqidx;
 865                 pi->first_nm_txq = nm_tqidx;
 866                 if (is_10G_port(pi) || is_40G_port(pi)) {
 867                         pi->nnmrxq = iaq.nnmrxq10g;
 868                         pi->nnmtxq = iaq.nnmtxq10g;
 869                 } else {
 870                         pi->nnmrxq = iaq.nnmrxq1g;
 871                         pi->nnmtxq = iaq.nnmtxq1g;
 872                 }
 873                 nm_rqidx += pi->nnmrxq;
 874                 nm_tqidx += pi->nnmtxq;
 875 #endif
 876         }
 877
 878         rc = setup_intr_handlers(sc);
 879         if (rc != 0) {
 880                 device_printf(dev,
 881                     "failed to setup interrupt handlers: %d\n", rc);
 882                 goto done;
 883         }
 884
 885         rc = bus_generic_attach(dev);
 886         if (rc != 0) {
 887                 device_printf(dev,
 888                     "failed to attach all child ports: %d\n", rc);
 889                 goto done;
 890         }
 891
 892         switch (sc->params.pci.speed) {
 893                 case 0x1:
 894                         pcie_ts = "2.5";
 895                         break;
 896                 case 0x2:
 897                         pcie_ts = "5.0";
 898                         break;
 899                 case 0x3:
 900                         pcie_ts = "8.0";
 901                         break;
 902                 default:
 903                         pcie_ts = "??";
 904                         break;
 905         }
 906         device_printf(dev,
 907             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 908             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 909             sc->params.nports, sc->intr_count,
 910             sc->intr_type == INTR_MSIX ? "MSI-X" :
 911             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 912             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 913
 914         t4_set_desc(sc);
 915
 916 done:
 917         if (rc != 0 && sc->cdev) {
 918                 /* cdev was created and so cxgbetool works; recover that way. */
 919                 device_printf(dev,
 920                     "error during attach, adapter is now in recovery mode.\n");
 921                 rc = 0;
 922         }
 923
 924         if (rc != 0)
 925                 t4_detach(dev);
 926         else
 927                 t4_sysctls(sc);
 928
 929         return (rc);
 930 }
 931
 932 /*
 933  * Idempotent
 934  */
 935 static int
 936 t4_detach(device_t dev)
 937 {
 938         struct adapter *sc;
 939         struct port_info *pi;
 940         int i, rc;
 941
 942         sc = device_get_softc(dev);
 943
 944         if (sc->flags & FULL_INIT_DONE)
 945                 t4_intr_disable(sc);
 946
 947         if (sc->cdev) {
 948                 destroy_dev(sc->cdev);
 949                 sc->cdev = NULL;
 950         }
 951
 952         rc = bus_generic_detach(dev);
 953         if (rc) {
 954                 device_printf(dev,
 955                     "failed to detach child devices: %d\n", rc);
 956                 return (rc);
 957         }
 958
 959         for (i = 0; i < sc->intr_count; i++)
 960                 t4_free_irq(sc, &sc->irq[i]);
 961
 962         for (i = 0; i < MAX_NPORTS; i++) {
 963                 pi = sc->port[i];
 964                 if (pi) {
 965                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 966                         if (pi->dev)
 967                                 device_delete_child(dev, pi->dev);
 968
 969                         mtx_destroy(&pi->pi_lock);
 970                         free(pi, M_CXGBE);
 971                 }
 972         }
 973
 974         if (sc->flags & FULL_INIT_DONE)
 975                 adapter_full_uninit(sc);
 976
 977         if (sc->flags & FW_OK)
 978                 t4_fw_bye(sc, sc->mbox);
 979
 980         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 981                 pci_release_msi(dev);
 982
 983         if (sc->regs_res)
 984                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 985                     sc->regs_res);
 986
 987         if (sc->udbs_res)
 988                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 989                     sc->udbs_res);
 990
 991         if (sc->msix_res)
 992                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 993                     sc->msix_res);
 994
 995         if (sc->l2t)
 996                 t4_free_l2t(sc->l2t);
 997
 998 #ifdef TCP_OFFLOAD
 999         free(sc->sge.ofld_rxq, M_CXGBE);
1000         free(sc->sge.ofld_txq, M_CXGBE);
1001 #endif
1002 #ifdef DEV_NETMAP
1003         free(sc->sge.nm_rxq, M_CXGBE);
1004         free(sc->sge.nm_txq, M_CXGBE);
1005 #endif
1006         free(sc->irq, M_CXGBE);
1007         free(sc->sge.rxq, M_CXGBE);
1008         free(sc->sge.txq, M_CXGBE);
1009         free(sc->sge.ctrlq, M_CXGBE);
1010         free(sc->sge.iqmap, M_CXGBE);
1011         free(sc->sge.eqmap, M_CXGBE);
1012         free(sc->tids.ftid_tab, M_CXGBE);
1013         t4_destroy_dma_tag(sc);
1014         if (mtx_initialized(&sc->sc_lock)) {
1015                 sx_xlock(&t4_list_lock);
1016                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1017                 sx_xunlock(&t4_list_lock);
1018                 mtx_destroy(&sc->sc_lock);
1019         }
1020
1021         if (mtx_initialized(&sc->tids.ftid_lock))
1022                 mtx_destroy(&sc->tids.ftid_lock);
1023         if (mtx_initialized(&sc->sfl_lock))
1024                 mtx_destroy(&sc->sfl_lock);
1025         if (mtx_initialized(&sc->ifp_lock))
1026                 mtx_destroy(&sc->ifp_lock);
1027
1028         bzero(sc, sizeof(*sc));
1029
1030         return (0);
1031 }
1032
1033 static int
1034 cxgbe_probe(device_t dev)
1035 {
1036         char buf[128];
1037         struct port_info *pi = device_get_softc(dev);
1038
1039         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1040         device_set_desc_copy(dev, buf);
1041
1042         return (BUS_PROBE_DEFAULT);
1043 }
1044
1045 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1046     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1047     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1048 #define T4_CAP_ENABLE (T4_CAP)
1049
1050 static int
1051 cxgbe_attach(device_t dev)
1052 {
1053         struct port_info *pi = device_get_softc(dev);
1054         struct ifnet *ifp;
1055         char *s;
1056         int n, o;
1057
1058         /* Allocate an ifnet and set it up */
1059         ifp = if_alloc(IFT_ETHER);
1060         if (ifp == NULL) {
1061                 device_printf(dev, "Cannot allocate ifnet\n");
1062                 return (ENOMEM);
1063         }
1064         pi->ifp = ifp;
1065         ifp->if_softc = pi;
1066
1067         callout_init(&pi->tick, CALLOUT_MPSAFE);
1068
1069         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1070         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1071
1072         ifp->if_init = cxgbe_init;
1073         ifp->if_ioctl = cxgbe_ioctl;
1074         ifp->if_transmit = cxgbe_transmit;
1075         ifp->if_qflush = cxgbe_qflush;
1076
1077         ifp->if_capabilities = T4_CAP;
1078 #ifdef TCP_OFFLOAD
1079         if (is_offload(pi->adapter))
1080                 ifp->if_capabilities |= IFCAP_TOE;
1081 #endif
1082         ifp->if_capenable = T4_CAP_ENABLE;
1083         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1084             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1085
1086         /* Initialize ifmedia for this port */
1087         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1088             cxgbe_media_status);
1089         build_medialist(pi, &pi->media);
1090
1091         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1092             EVENTHANDLER_PRI_ANY);
1093
1094         ether_ifattach(ifp, pi->hw_addr);
1095
1096         n = 128;
1097         s = malloc(n, M_CXGBE, M_WAITOK);
1098         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1099         MPASS(n > o);
1100 #ifdef TCP_OFFLOAD
1101         if (is_offload(pi->adapter)) {
1102                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1103                     pi->nofldtxq, pi->nofldrxq);
1104                 MPASS(n > o);
1105         }
1106 #endif
1107 #ifdef DEV_NETMAP
1108         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1109             pi->nnmrxq);
1110         MPASS(n > o);
1111 #endif
1112         device_printf(dev, "%s\n", s);
1113         free(s, M_CXGBE);
1114
1115 #ifdef DEV_NETMAP
1116         /* nm_media handled here to keep implementation private to this file */
1117         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1118             cxgbe_media_status);
1119         build_medialist(pi, &pi->nm_media);
1120         create_netmap_ifnet(pi);        /* logs errors it something fails */
1121 #endif
1122         cxgbe_sysctls(pi);
1123
1124         return (0);
1125 }
1126
1127 static int
1128 cxgbe_detach(device_t dev)
1129 {
1130         struct port_info *pi = device_get_softc(dev);
1131         struct adapter *sc = pi->adapter;
1132         struct ifnet *ifp = pi->ifp;
1133
1134         /* Tell if_ioctl and if_init that the port is going away */
1135         ADAPTER_LOCK(sc);
1136         SET_DOOMED(pi);
1137         wakeup(&sc->flags);
1138         while (IS_BUSY(sc))
1139                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1140         SET_BUSY(sc);
1141 #ifdef INVARIANTS
1142         sc->last_op = "t4detach";
1143         sc->last_op_thr = curthread;
1144 #endif
1145         ADAPTER_UNLOCK(sc);
1146
1147         if (pi->flags & HAS_TRACEQ) {
1148                 sc->traceq = -1;        /* cloner should not create ifnet */
1149                 t4_tracer_port_detach(sc);
1150         }
1151
1152         if (pi->vlan_c)
1153                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1154
1155         PORT_LOCK(pi);
1156         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1157         callout_stop(&pi->tick);
1158         PORT_UNLOCK(pi);
1159         callout_drain(&pi->tick);
1160
1161         /* Let detach proceed even if these fail. */
1162         cxgbe_uninit_synchronized(pi);
1163         port_full_uninit(pi);
1164
1165         ifmedia_removeall(&pi->media);
1166         ether_ifdetach(pi->ifp);
1167         if_free(pi->ifp);
1168
1169 #ifdef DEV_NETMAP
1170         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1171         destroy_netmap_ifnet(pi);
1172 #endif
1173
1174         ADAPTER_LOCK(sc);
1175         CLR_BUSY(sc);
1176         wakeup(&sc->flags);
1177         ADAPTER_UNLOCK(sc);
1178
1179         return (0);
1180 }
1181
1182 static void
1183 cxgbe_init(void *arg)
1184 {
1185         struct port_info *pi = arg;
1186         struct adapter *sc = pi->adapter;
1187
1188         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1189                 return;
1190         cxgbe_init_synchronized(pi);
1191         end_synchronized_op(sc, 0);
1192 }
1193
1194 static int
1195 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1196 {
1197         int rc = 0, mtu, flags, can_sleep;
1198         struct port_info *pi = ifp->if_softc;
1199         struct adapter *sc = pi->adapter;
1200         struct ifreq *ifr = (struct ifreq *)data;
1201         uint32_t mask;
1202
1203         switch (cmd) {
1204         case SIOCSIFMTU:
1205                 mtu = ifr->ifr_mtu;
1206                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1207                         return (EINVAL);
1208
1209                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1210                 if (rc)
1211                         return (rc);
1212                 ifp->if_mtu = mtu;
1213                 if (pi->flags & PORT_INIT_DONE) {
1214                         t4_update_fl_bufsize(ifp);
1215                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1216                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1217                 }
1218                 end_synchronized_op(sc, 0);
1219                 break;
1220
1221         case SIOCSIFFLAGS:
1222                 can_sleep = 0;
1223 redo_sifflags:
1224                 rc = begin_synchronized_op(sc, pi,
1225                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1226                 if (rc)
1227                         return (rc);
1228
1229                 if (ifp->if_flags & IFF_UP) {
1230                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1231                                 flags = pi->if_flags;
1232                                 if ((ifp->if_flags ^ flags) &
1233                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1234                                         if (can_sleep == 1) {
1235                                                 end_synchronized_op(sc, 0);
1236                                                 can_sleep = 0;
1237                                                 goto redo_sifflags;
1238                                         }
1239                                         rc = update_mac_settings(ifp,
1240                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1241                                 }
1242                         } else {
1243                                 if (can_sleep == 0) {
1244                                         end_synchronized_op(sc, LOCK_HELD);
1245                                         can_sleep = 1;
1246                                         goto redo_sifflags;
1247                                 }
1248                                 rc = cxgbe_init_synchronized(pi);
1249                         }
1250                         pi->if_flags = ifp->if_flags;
1251                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1252                         if (can_sleep == 0) {
1253                                 end_synchronized_op(sc, LOCK_HELD);
1254                                 can_sleep = 1;
1255                                 goto redo_sifflags;
1256                         }
1257                         rc = cxgbe_uninit_synchronized(pi);
1258                 }
1259                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1260                 break;
1261
1262         case SIOCADDMULTI:
1263         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1264                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1265                 if (rc)
1266                         return (rc);
1267                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1268                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1269                 end_synchronized_op(sc, LOCK_HELD);
1270                 break;
1271
1272         case SIOCSIFCAP:
1273                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1274                 if (rc)
1275                         return (rc);
1276
1277                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1278                 if (mask & IFCAP_TXCSUM) {
1279                         ifp->if_capenable ^= IFCAP_TXCSUM;
1280                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1281
1282                         if (IFCAP_TSO4 & ifp->if_capenable &&
1283                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1284                                 ifp->if_capenable &= ~IFCAP_TSO4;
1285                                 if_printf(ifp,
1286                                     "tso4 disabled due to -txcsum.\n");
1287                         }
1288                 }
1289                 if (mask & IFCAP_TXCSUM_IPV6) {
1290                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1291                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1292
1293                         if (IFCAP_TSO6 & ifp->if_capenable &&
1294                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1295                                 ifp->if_capenable &= ~IFCAP_TSO6;
1296                                 if_printf(ifp,
1297                                     "tso6 disabled due to -txcsum6.\n");
1298                         }
1299                 }
1300                 if (mask & IFCAP_RXCSUM)
1301                         ifp->if_capenable ^= IFCAP_RXCSUM;
1302                 if (mask & IFCAP_RXCSUM_IPV6)
1303                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1304
1305                 /*
1306                  * Note that we leave CSUM_TSO alone (it is always set).  The
1307                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1308                  * sending a TSO request our way, so it's sufficient to toggle
1309                  * IFCAP_TSOx only.
1310                  */
1311                 if (mask & IFCAP_TSO4) {
1312                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1313                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1314                                 if_printf(ifp, "enable txcsum first.\n");
1315                                 rc = EAGAIN;
1316                                 goto fail;
1317                         }
1318                         ifp->if_capenable ^= IFCAP_TSO4;
1319                 }
1320                 if (mask & IFCAP_TSO6) {
1321                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1322                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1323                                 if_printf(ifp, "enable txcsum6 first.\n");
1324                                 rc = EAGAIN;
1325                                 goto fail;
1326                         }
1327                         ifp->if_capenable ^= IFCAP_TSO6;
1328                 }
1329                 if (mask & IFCAP_LRO) {
1330 #if defined(INET) || defined(INET6)
1331                         int i;
1332                         struct sge_rxq *rxq;
1333
1334                         ifp->if_capenable ^= IFCAP_LRO;
1335                         for_each_rxq(pi, i, rxq) {
1336                                 if (ifp->if_capenable & IFCAP_LRO)
1337                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1338                                 else
1339                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1340                         }
1341 #endif
1342                 }
1343 #ifdef TCP_OFFLOAD
1344                 if (mask & IFCAP_TOE) {
1345                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1346
1347                         rc = toe_capability(pi, enable);
1348                         if (rc != 0)
1349                                 goto fail;
1350
1351                         ifp->if_capenable ^= mask;
1352                 }
1353 #endif
1354                 if (mask & IFCAP_VLAN_HWTAGGING) {
1355                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1356                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1357                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1358                 }
1359                 if (mask & IFCAP_VLAN_MTU) {
1360                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1361
1362                         /* Need to find out how to disable auto-mtu-inflation */
1363                 }
1364                 if (mask & IFCAP_VLAN_HWTSO)
1365                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1366                 if (mask & IFCAP_VLAN_HWCSUM)
1367                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1368
1369 #ifdef VLAN_CAPABILITIES
1370                 VLAN_CAPABILITIES(ifp);
1371 #endif
1372 fail:
1373                 end_synchronized_op(sc, 0);
1374                 break;
1375
1376         case SIOCSIFMEDIA:
1377         case SIOCGIFMEDIA:
1378                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1379                 break;
1380
1381         default:
1382                 rc = ether_ioctl(ifp, cmd, data);
1383         }
1384
1385         return (rc);
1386 }
1387
1388 static int
1389 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1390 {
1391         struct port_info *pi = ifp->if_softc;
1392         struct adapter *sc = pi->adapter;
1393         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1394         struct buf_ring *br;
1395         int rc;
1396
1397         M_ASSERTPKTHDR(m);
1398
1399         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1400                 m_freem(m);
1401                 return (ENETDOWN);
1402         }
1403
1404         if (m->m_flags & M_FLOWID)
1405                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1406                     + pi->rsrv_noflowq);
1407         br = txq->br;
1408
1409         if (TXQ_TRYLOCK(txq) == 0) {
1410                 struct sge_eq *eq = &txq->eq;
1411
1412                 /*
1413                  * It is possible that t4_eth_tx finishes up and releases the
1414                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1415                  * need to make sure that this mbuf doesn't just sit there in
1416                  * the drbr.
1417                  */
1418
1419                 rc = drbr_enqueue(ifp, br, m);
1420                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1421                     !(eq->flags & EQ_DOOMED))
1422                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1423                 return (rc);
1424         }
1425
1426         /*
1427          * txq->m is the mbuf that is held up due to a temporary shortage of
1428          * resources and it should be put on the wire first.  Then what's in
1429          * drbr and finally the mbuf that was just passed in to us.
1430          *
1431          * Return code should indicate the fate of the mbuf that was passed in
1432          * this time.
1433          */
1434
1435         TXQ_LOCK_ASSERT_OWNED(txq);
1436         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1437
1438                 /* Queued for transmission. */
1439
1440                 rc = drbr_enqueue(ifp, br, m);
1441                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1442                 (void) t4_eth_tx(ifp, txq, m);
1443                 TXQ_UNLOCK(txq);
1444                 return (rc);
1445         }
1446
1447         /* Direct transmission. */
1448         rc = t4_eth_tx(ifp, txq, m);
1449         if (rc != 0 && txq->m)
1450                 rc = 0; /* held, will be transmitted soon (hopefully) */
1451
1452         TXQ_UNLOCK(txq);
1453         return (rc);
1454 }
1455
1456 static void
1457 cxgbe_qflush(struct ifnet *ifp)
1458 {
1459         struct port_info *pi = ifp->if_softc;
1460         struct sge_txq *txq;
1461         int i;
1462         struct mbuf *m;
1463
1464         /* queues do not exist if !PORT_INIT_DONE. */
1465         if (pi->flags & PORT_INIT_DONE) {
1466                 for_each_txq(pi, i, txq) {
1467                         TXQ_LOCK(txq);
1468                         m_freem(txq->m);
1469                         txq->m = NULL;
1470                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1471                                 m_freem(m);
1472                         TXQ_UNLOCK(txq);
1473                 }
1474         }
1475         if_qflush(ifp);
1476 }
1477
1478 static int
1479 cxgbe_media_change(struct ifnet *ifp)
1480 {
1481         struct port_info *pi = ifp->if_softc;
1482
1483         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1484
1485         return (EOPNOTSUPP);
1486 }
1487
1488 static void
1489 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1490 {
1491         struct port_info *pi = ifp->if_softc;
1492         struct ifmedia *media = NULL;
1493         struct ifmedia_entry *cur;
1494         int speed = pi->link_cfg.speed;
1495         int data = (pi->port_type << 8) | pi->mod_type;
1496
1497         if (ifp == pi->ifp)
1498                 media = &pi->media;
1499 #ifdef DEV_NETMAP
1500         else if (ifp == pi->nm_ifp)
1501                 media = &pi->nm_media;
1502 #endif
1503         MPASS(media != NULL);
1504
1505         cur = media->ifm_cur;
1506         if (cur->ifm_data != data) {
1507                 build_medialist(pi, media);
1508                 cur = media->ifm_cur;
1509         }
1510
1511         ifmr->ifm_status = IFM_AVALID;
1512         if (!pi->link_cfg.link_ok)
1513                 return;
1514
1515         ifmr->ifm_status |= IFM_ACTIVE;
1516
1517         /* active and current will differ iff current media is autoselect. */
1518         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1519                 return;
1520
1521         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1522         if (speed == SPEED_10000)
1523                 ifmr->ifm_active |= IFM_10G_T;
1524         else if (speed == SPEED_1000)
1525                 ifmr->ifm_active |= IFM_1000_T;
1526         else if (speed == SPEED_100)
1527                 ifmr->ifm_active |= IFM_100_TX;
1528         else if (speed == SPEED_10)
1529                 ifmr->ifm_active |= IFM_10_T;
1530         else
1531                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1532                             speed));
1533 }
1534
1535 void
1536 t4_fatal_err(struct adapter *sc)
1537 {
1538         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1539         t4_intr_disable(sc);
1540         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1541             device_get_nameunit(sc->dev));
1542 }
1543
1544 static int
1545 map_bars_0_and_4(struct adapter *sc)
1546 {
1547         sc->regs_rid = PCIR_BAR(0);
1548         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1549             &sc->regs_rid, RF_ACTIVE);
1550         if (sc->regs_res == NULL) {
1551                 device_printf(sc->dev, "cannot map registers.\n");
1552                 return (ENXIO);
1553         }
1554         sc->bt = rman_get_bustag(sc->regs_res);
1555         sc->bh = rman_get_bushandle(sc->regs_res);
1556         sc->mmio_len = rman_get_size(sc->regs_res);
1557         setbit(&sc->doorbells, DOORBELL_KDB);
1558
1559         sc->msix_rid = PCIR_BAR(4);
1560         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1561             &sc->msix_rid, RF_ACTIVE);
1562         if (sc->msix_res == NULL) {
1563                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1564                 return (ENXIO);
1565         }
1566
1567         return (0);
1568 }
1569
1570 static int
1571 map_bar_2(struct adapter *sc)
1572 {
1573
1574         /*
1575          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1576          * to map it if RDMA is disabled.
1577          */
1578         if (is_t4(sc) && sc->rdmacaps == 0)
1579                 return (0);
1580
1581         sc->udbs_rid = PCIR_BAR(2);
1582         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1583             &sc->udbs_rid, RF_ACTIVE);
1584         if (sc->udbs_res == NULL) {
1585                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1586                 return (ENXIO);
1587         }
1588         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1589
1590         if (is_t5(sc)) {
1591                 setbit(&sc->doorbells, DOORBELL_UDB);
1592 #if defined(__i386__) || defined(__amd64__)
1593                 if (t5_write_combine) {
1594                         int rc;
1595
1596                         /*
1597                          * Enable write combining on BAR2.  This is the
1598                          * userspace doorbell BAR and is split into 128B
1599                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1600                          * with an egress queue.  The first 64B has the doorbell
1601                          * and the second 64B can be used to submit a tx work
1602                          * request with an implicit doorbell.
1603                          */
1604
1605                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1606                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1607                         if (rc == 0) {
1608                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1609                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1610                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1611                         } else {
1612                                 device_printf(sc->dev,
1613                                     "couldn't enable write combining: %d\n",
1614                                     rc);
1615                         }
1616
1617                         t4_write_reg(sc, A_SGE_STAT_CFG,
1618                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1619                 }
1620 #endif
1621         }
1622
1623         return (0);
1624 }
1625
1626 static const struct memwin t4_memwin[] = {
1627         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1628         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1629         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1630 };
1631
1632 static const struct memwin t5_memwin[] = {
1633         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1634         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1635         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1636 };
1637
1638 static void
1639 setup_memwin(struct adapter *sc)
1640 {
1641         const struct memwin *mw;
1642         int i, n;
1643         uint32_t bar0;
1644
1645         if (is_t4(sc)) {
1646                 /*
1647                  * Read low 32b of bar0 indirectly via the hardware backdoor
1648                  * mechanism.  Works from within PCI passthrough environments
1649                  * too, where rman_get_start() can return a different value.  We
1650                  * need to program the T4 memory window decoders with the actual
1651                  * addresses that will be coming across the PCIe link.
1652                  */
1653                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1654                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1655
1656                 mw = &t4_memwin[0];
1657                 n = nitems(t4_memwin);
1658         } else {
1659                 /* T5 uses the relative offset inside the PCIe BAR */
1660                 bar0 = 0;
1661
1662                 mw = &t5_memwin[0];
1663                 n = nitems(t5_memwin);
1664         }
1665
1666         for (i = 0; i < n; i++, mw++) {
1667                 t4_write_reg(sc,
1668                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1669                     (mw->base + bar0) | V_BIR(0) |
1670                     V_WINDOW(ilog2(mw->aperture) - 10));
1671         }
1672
1673         /* flush */
1674         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1675 }
1676
1677 /*
1678  * Verify that the memory range specified by the addr/len pair is valid and lies
1679  * entirely within a single region (EDCx or MCx).
1680  */
1681 static int
1682 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1683 {
1684         uint32_t em, addr_len, maddr, mlen;
1685
1686         /* Memory can only be accessed in naturally aligned 4 byte units */
1687         if (addr & 3 || len & 3 || len == 0)
1688                 return (EINVAL);
1689
1690         /* Enabled memories */
1691         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1692         if (em & F_EDRAM0_ENABLE) {
1693                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1694                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1695                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1696                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1697                     addr + len <= maddr + mlen)
1698                         return (0);
1699         }
1700         if (em & F_EDRAM1_ENABLE) {
1701                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1702                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1703                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1704                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1705                     addr + len <= maddr + mlen)
1706                         return (0);
1707         }
1708         if (em & F_EXT_MEM_ENABLE) {
1709                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1710                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1711                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1712                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1713                     addr + len <= maddr + mlen)
1714                         return (0);
1715         }
1716         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1717                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1718                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1719                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1720                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1721                     addr + len <= maddr + mlen)
1722                         return (0);
1723         }
1724
1725         return (EFAULT);
1726 }
1727
1728 static int
1729 fwmtype_to_hwmtype(int mtype)
1730 {
1731
1732         switch (mtype) {
1733         case FW_MEMTYPE_EDC0:
1734                 return (MEM_EDC0);
1735         case FW_MEMTYPE_EDC1:
1736                 return (MEM_EDC1);
1737         case FW_MEMTYPE_EXTMEM:
1738                 return (MEM_MC0);
1739         case FW_MEMTYPE_EXTMEM1:
1740                 return (MEM_MC1);
1741         default:
1742                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1743         }
1744 }
1745
1746 /*
1747  * Verify that the memory range specified by the memtype/offset/len pair is
1748  * valid and lies entirely within the memtype specified.  The global address of
1749  * the start of the range is returned in addr.
1750  */
1751 static int
1752 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1753     uint32_t *addr)
1754 {
1755         uint32_t em, addr_len, maddr, mlen;
1756
1757         /* Memory can only be accessed in naturally aligned 4 byte units */
1758         if (off & 3 || len & 3 || len == 0)
1759                 return (EINVAL);
1760
1761         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1762         switch (fwmtype_to_hwmtype(mtype)) {
1763         case MEM_EDC0:
1764                 if (!(em & F_EDRAM0_ENABLE))
1765                         return (EINVAL);
1766                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1767                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1768                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1769                 break;
1770         case MEM_EDC1:
1771                 if (!(em & F_EDRAM1_ENABLE))
1772                         return (EINVAL);
1773                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1774                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1775                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1776                 break;
1777         case MEM_MC:
1778                 if (!(em & F_EXT_MEM_ENABLE))
1779                         return (EINVAL);
1780                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1781                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1782                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1783                 break;
1784         case MEM_MC1:
1785                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1786                         return (EINVAL);
1787                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1788                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1789                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1790                 break;
1791         default:
1792                 return (EINVAL);
1793         }
1794
1795         if (mlen > 0 && off < mlen && off + len <= mlen) {
1796                 *addr = maddr + off;    /* global address */
1797                 return (0);
1798         }
1799
1800         return (EFAULT);
1801 }
1802
1803 static void
1804 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1805 {
1806         const struct memwin *mw;
1807
1808         if (is_t4(sc)) {
1809                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1810                     ("%s: incorrect memwin# (%d)", __func__, win));
1811                 mw = &t4_memwin[win];
1812         } else {
1813                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1814                     ("%s: incorrect memwin# (%d)", __func__, win));
1815                 mw = &t5_memwin[win];
1816         }
1817
1818         if (base != NULL)
1819                 *base = mw->base;
1820         if (aperture != NULL)
1821                 *aperture = mw->aperture;
1822 }
1823
1824 /*
1825  * Positions the memory window such that it can be used to access the specified
1826  * address in the chip's address space.  The return value is the offset of addr
1827  * from the start of the window.
1828  */
1829 static uint32_t
1830 position_memwin(struct adapter *sc, int n, uint32_t addr)
1831 {
1832         uint32_t start, pf;
1833         uint32_t reg;
1834
1835         KASSERT(n >= 0 && n <= 3,
1836             ("%s: invalid window %d.", __func__, n));
1837         KASSERT((addr & 3) == 0,
1838             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1839
1840         if (is_t4(sc)) {
1841                 pf = 0;
1842                 start = addr & ~0xf;    /* start must be 16B aligned */
1843         } else {
1844                 pf = V_PFNUM(sc->pf);
1845                 start = addr & ~0x7f;   /* start must be 128B aligned */
1846         }
1847         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1848
1849         t4_write_reg(sc, reg, start | pf);
1850         t4_read_reg(sc, reg);
1851
1852         return (addr - start);
1853 }
1854
1855 static int
1856 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1857     struct intrs_and_queues *iaq)
1858 {
1859         int rc, itype, navail, nrxq10g, nrxq1g, n;
1860         int nofldrxq10g = 0, nofldrxq1g = 0;
1861         int nnmrxq10g = 0, nnmrxq1g = 0;
1862
1863         bzero(iaq, sizeof(*iaq));
1864
1865         iaq->ntxq10g = t4_ntxq10g;
1866         iaq->ntxq1g = t4_ntxq1g;
1867         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1868         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1869         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1870 #ifdef TCP_OFFLOAD
1871         if (is_offload(sc)) {
1872                 iaq->nofldtxq10g = t4_nofldtxq10g;
1873                 iaq->nofldtxq1g = t4_nofldtxq1g;
1874                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1875                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1876         }
1877 #endif
1878 #ifdef DEV_NETMAP
1879         iaq->nnmtxq10g = t4_nnmtxq10g;
1880         iaq->nnmtxq1g = t4_nnmtxq1g;
1881         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1882         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1883 #endif
1884
1885         for (itype = INTR_MSIX; itype; itype >>= 1) {
1886
1887                 if ((itype & t4_intr_types) == 0)
1888                         continue;       /* not allowed */
1889
1890                 if (itype == INTR_MSIX)
1891                         navail = pci_msix_count(sc->dev);
1892                 else if (itype == INTR_MSI)
1893                         navail = pci_msi_count(sc->dev);
1894                 else
1895                         navail = 1;
1896 restart:
1897                 if (navail == 0)
1898                         continue;
1899
1900                 iaq->intr_type = itype;
1901                 iaq->intr_flags_10g = 0;
1902                 iaq->intr_flags_1g = 0;
1903
1904                 /*
1905                  * Best option: an interrupt vector for errors, one for the
1906                  * firmware event queue, and one for every rxq (NIC, TOE, and
1907                  * netmap).
1908                  */
1909                 iaq->nirq = T4_EXTRA_INTR;
1910                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1911                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1912                 if (iaq->nirq <= navail &&
1913                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1914                         iaq->intr_flags_10g = INTR_ALL;
1915                         iaq->intr_flags_1g = INTR_ALL;
1916                         goto allocate;
1917                 }
1918
1919                 /*
1920                  * Second best option: a vector for errors, one for the firmware
1921                  * event queue, and vectors for either all the NIC rx queues or
1922                  * all the TOE rx queues.  The queues that don't get vectors
1923                  * will forward their interrupts to those that do.
1924                  *
1925                  * Note: netmap rx queues cannot be created early and so they
1926                  * can't be setup to receive forwarded interrupts for others.
1927                  */
1928                 iaq->nirq = T4_EXTRA_INTR;
1929                 if (nrxq10g >= nofldrxq10g) {
1930                         iaq->intr_flags_10g = INTR_RXQ;
1931                         iaq->nirq += n10g * nrxq10g;
1932 #ifdef DEV_NETMAP
1933                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1934 #endif
1935                 } else {
1936                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1937                         iaq->nirq += n10g * nofldrxq10g;
1938 #ifdef DEV_NETMAP
1939                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1940 #endif
1941                 }
1942                 if (nrxq1g >= nofldrxq1g) {
1943                         iaq->intr_flags_1g = INTR_RXQ;
1944                         iaq->nirq += n1g * nrxq1g;
1945 #ifdef DEV_NETMAP
1946                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1947 #endif
1948                 } else {
1949                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1950                         iaq->nirq += n1g * nofldrxq1g;
1951 #ifdef DEV_NETMAP
1952                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1953 #endif
1954                 }
1955                 if (iaq->nirq <= navail &&
1956                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1957                         goto allocate;
1958
1959                 /*
1960                  * Next best option: an interrupt vector for errors, one for the
1961                  * firmware event queue, and at least one per port.  At this
1962                  * point we know we'll have to downsize nrxq and/or nofldrxq
1963                  * and/or nnmrxq to fit what's available to us.
1964                  */
1965                 iaq->nirq = T4_EXTRA_INTR;
1966                 iaq->nirq += n10g + n1g;
1967                 if (iaq->nirq <= navail) {
1968                         int leftover = navail - iaq->nirq;
1969
1970                         if (n10g > 0) {
1971                                 int target = max(nrxq10g, nofldrxq10g);
1972
1973                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1974                                     INTR_RXQ : INTR_OFLD_RXQ;
1975
1976                                 n = 1;
1977                                 while (n < target && leftover >= n10g) {
1978                                         leftover -= n10g;
1979                                         iaq->nirq += n10g;
1980                                         n++;
1981                                 }
1982                                 iaq->nrxq10g = min(n, nrxq10g);
1983 #ifdef TCP_OFFLOAD
1984                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1985 #endif
1986 #ifdef DEV_NETMAP
1987                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1988 #endif
1989                         }
1990
1991                         if (n1g > 0) {
1992                                 int target = max(nrxq1g, nofldrxq1g);
1993
1994                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1995                                     INTR_RXQ : INTR_OFLD_RXQ;
1996
1997                                 n = 1;
1998                                 while (n < target && leftover >= n1g) {
1999                                         leftover -= n1g;
2000                                         iaq->nirq += n1g;
2001                                         n++;
2002                                 }
2003                                 iaq->nrxq1g = min(n, nrxq1g);
2004 #ifdef TCP_OFFLOAD
2005                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2006 #endif
2007 #ifdef DEV_NETMAP
2008                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2009 #endif
2010                         }
2011
2012                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2013                                 goto allocate;
2014                 }
2015
2016                 /*
2017                  * Least desirable option: one interrupt vector for everything.
2018                  */
2019                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2020                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2021 #ifdef TCP_OFFLOAD
2022                 if (is_offload(sc))
2023                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2024 #endif
2025 #ifdef DEV_NETMAP
2026                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2027 #endif
2028
2029 allocate:
2030                 navail = iaq->nirq;
2031                 rc = 0;
2032                 if (itype == INTR_MSIX)
2033                         rc = pci_alloc_msix(sc->dev, &navail);
2034                 else if (itype == INTR_MSI)
2035                         rc = pci_alloc_msi(sc->dev, &navail);
2036
2037                 if (rc == 0) {
2038                         if (navail == iaq->nirq)
2039                                 return (0);
2040
2041                         /*
2042                          * Didn't get the number requested.  Use whatever number
2043                          * the kernel is willing to allocate (it's in navail).
2044                          */
2045                         device_printf(sc->dev, "fewer vectors than requested, "
2046                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2047                             itype, iaq->nirq, navail);
2048                         pci_release_msi(sc->dev);
2049                         goto restart;
2050                 }
2051
2052                 device_printf(sc->dev,
2053                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2054                     itype, rc, iaq->nirq, navail);
2055         }
2056
2057         device_printf(sc->dev,
2058             "failed to find a usable interrupt type.  "
2059             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2060             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2061
2062         return (ENXIO);
2063 }
2064
2065 #define FW_VERSION(chip) ( \
2066     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2067     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2068     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2069     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2070 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2071
2072 struct fw_info {
2073         uint8_t chip;
2074         char *kld_name;
2075         char *fw_mod_name;
2076         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2077 } fw_info[] = {
2078         {
2079                 .chip = CHELSIO_T4,
2080                 .kld_name = "t4fw_cfg",
2081                 .fw_mod_name = "t4fw",
2082                 .fw_hdr = {
2083                         .chip = FW_HDR_CHIP_T4,
2084                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2085                         .intfver_nic = FW_INTFVER(T4, NIC),
2086                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2087                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2088                         .intfver_ri = FW_INTFVER(T4, RI),
2089                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2090                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2091                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2092                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2093                 },
2094         }, {
2095                 .chip = CHELSIO_T5,
2096                 .kld_name = "t5fw_cfg",
2097                 .fw_mod_name = "t5fw",
2098                 .fw_hdr = {
2099                         .chip = FW_HDR_CHIP_T5,
2100                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2101                         .intfver_nic = FW_INTFVER(T5, NIC),
2102                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2103                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2104                         .intfver_ri = FW_INTFVER(T5, RI),
2105                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2106                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2107                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2108                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2109                 },
2110         }
2111 };
2112
2113 static struct fw_info *
2114 find_fw_info(int chip)
2115 {
2116         int i;
2117
2118         for (i = 0; i < nitems(fw_info); i++) {
2119                 if (fw_info[i].chip == chip)
2120                         return (&fw_info[i]);
2121         }
2122         return (NULL);
2123 }
2124
2125 /*
2126  * Is the given firmware API compatible with the one the driver was compiled
2127  * with?
2128  */
2129 static int
2130 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2131 {
2132
2133         /* short circuit if it's the exact same firmware version */
2134         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2135                 return (1);
2136
2137         /*
2138          * XXX: Is this too conservative?  Perhaps I should limit this to the
2139          * features that are supported in the driver.
2140          */
2141 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2142         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2143             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2144             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2145                 return (1);
2146 #undef SAME_INTF
2147
2148         return (0);
2149 }
2150
2151 /*
2152  * The firmware in the KLD is usable, but should it be installed?  This routine
2153  * explains itself in detail if it indicates the KLD firmware should be
2154  * installed.
2155  */
2156 static int
2157 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2158 {
2159         const char *reason;
2160
2161         if (!card_fw_usable) {
2162                 reason = "incompatible or unusable";
2163                 goto install;
2164         }
2165
2166         if (k > c) {
2167                 reason = "older than the version bundled with this driver";
2168                 goto install;
2169         }
2170
2171         if (t4_fw_install == 2 && k != c) {
2172                 reason = "different than the version bundled with this driver";
2173                 goto install;
2174         }
2175
2176         return (0);
2177
2178 install:
2179         if (t4_fw_install == 0) {
2180                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2181                     "but the driver is prohibited from installing a different "
2182                     "firmware on the card.\n",
2183                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2184                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2185
2186                 return (0);
2187         }
2188
2189         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2190             "installing firmware %u.%u.%u.%u on card.\n",
2191             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2192             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2193             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2194             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2195
2196         return (1);
2197 }
2198 /*
2199  * Establish contact with the firmware and determine if we are the master driver
2200  * or not, and whether we are responsible for chip initialization.
2201  */
2202 static int
2203 prep_firmware(struct adapter *sc)
2204 {
2205         const struct firmware *fw = NULL, *default_cfg;
2206         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2207         enum dev_state state;
2208         struct fw_info *fw_info;
2209         struct fw_hdr *card_fw;         /* fw on the card */
2210         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2211         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2212                                            against */
2213
2214         /* Contact firmware. */
2215         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2216         if (rc < 0 || state == DEV_STATE_ERR) {
2217                 rc = -rc;
2218                 device_printf(sc->dev,
2219                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2220                 return (rc);
2221         }
2222         pf = rc;
2223         if (pf == sc->mbox)
2224                 sc->flags |= MASTER_PF;
2225         else if (state == DEV_STATE_UNINIT) {
2226                 /*
2227                  * We didn't get to be the master so we definitely won't be
2228                  * configuring the chip.  It's a bug if someone else hasn't
2229                  * configured it already.
2230                  */
2231                 device_printf(sc->dev, "couldn't be master(%d), "
2232                     "device not already initialized either(%d).\n", rc, state);
2233                 return (EDOOFUS);
2234         }
2235
2236         /* This is the firmware whose headers the driver was compiled against */
2237         fw_info = find_fw_info(chip_id(sc));
2238         if (fw_info == NULL) {
2239                 device_printf(sc->dev,
2240                     "unable to look up firmware information for chip %d.\n",
2241                     chip_id(sc));
2242                 return (EINVAL);
2243         }
2244         drv_fw = &fw_info->fw_hdr;
2245
2246         /*
2247          * The firmware KLD contains many modules.  The KLD name is also the
2248          * name of the module that contains the default config file.
2249          */
2250         default_cfg = firmware_get(fw_info->kld_name);
2251
2252         /* Read the header of the firmware on the card */
2253         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2254         rc = -t4_read_flash(sc, FLASH_FW_START,
2255             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2256         if (rc == 0)
2257                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2258         else {
2259                 device_printf(sc->dev,
2260                     "Unable to read card's firmware header: %d\n", rc);
2261                 card_fw_usable = 0;
2262         }
2263
2264         /* This is the firmware in the KLD */
2265         fw = firmware_get(fw_info->fw_mod_name);
2266         if (fw != NULL) {
2267                 kld_fw = (const void *)fw->data;
2268                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2269         } else {
2270                 kld_fw = NULL;
2271                 kld_fw_usable = 0;
2272         }
2273
2274         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2275             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2276                 /*
2277                  * Common case: the firmware on the card is an exact match and
2278                  * the KLD is an exact match too, or the KLD is
2279                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2280                  * here -- use cxgbetool loadfw if you want to reinstall the
2281                  * same firmware as the one on the card.
2282                  */
2283         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2284             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2285             be32toh(card_fw->fw_ver))) {
2286
2287                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2288                 if (rc != 0) {
2289                         device_printf(sc->dev,
2290                             "failed to install firmware: %d\n", rc);
2291                         goto done;
2292                 }
2293
2294                 /* Installed successfully, update the cached header too. */
2295                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2296                 card_fw_usable = 1;
2297                 need_fw_reset = 0;      /* already reset as part of load_fw */
2298         }
2299
2300         if (!card_fw_usable) {
2301                 uint32_t d, c, k;
2302
2303                 d = ntohl(drv_fw->fw_ver);
2304                 c = ntohl(card_fw->fw_ver);
2305                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2306
2307                 device_printf(sc->dev, "Cannot find a usable firmware: "
2308                     "fw_install %d, chip state %d, "
2309                     "driver compiled with %d.%d.%d.%d, "
2310                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2311                     t4_fw_install, state,
2312                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2313                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2314                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2315                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2316                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2317                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2318                 rc = EINVAL;
2319                 goto done;
2320         }
2321
2322         /* We're using whatever's on the card and it's known to be good. */
2323         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2324         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2325             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2326             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2327             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2328             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2329         t4_get_tp_version(sc, &sc->params.tp_vers);
2330
2331         /* Reset device */
2332         if (need_fw_reset &&
2333             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2334                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2335                 if (rc != ETIMEDOUT && rc != EIO)
2336                         t4_fw_bye(sc, sc->mbox);
2337                 goto done;
2338         }
2339         sc->flags |= FW_OK;
2340
2341         rc = get_params__pre_init(sc);
2342         if (rc != 0)
2343                 goto done; /* error message displayed already */
2344
2345         /* Partition adapter resources as specified in the config file. */
2346         if (state == DEV_STATE_UNINIT) {
2347
2348                 KASSERT(sc->flags & MASTER_PF,
2349                     ("%s: trying to change chip settings when not master.",
2350                     __func__));
2351
2352                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2353                 if (rc != 0)
2354                         goto done;      /* error message displayed already */
2355
2356                 t4_tweak_chip_settings(sc);
2357
2358                 /* get basic stuff going */
2359                 rc = -t4_fw_initialize(sc, sc->mbox);
2360                 if (rc != 0) {
2361                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2362                         goto done;
2363                 }
2364         } else {
2365                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2366                 sc->cfcsum = 0;
2367         }
2368
2369 done:
2370         free(card_fw, M_CXGBE);
2371         if (fw != NULL)
2372                 firmware_put(fw, FIRMWARE_UNLOAD);
2373         if (default_cfg != NULL)
2374                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2375
2376         return (rc);
2377 }
2378
2379 #define FW_PARAM_DEV(param) \
2380         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2381          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2382 #define FW_PARAM_PFVF(param) \
2383         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2384          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2385
2386 /*
2387  * Partition chip resources for use between various PFs, VFs, etc.
2388  */
2389 static int
2390 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2391     const char *name_prefix)
2392 {
2393         const struct firmware *cfg = NULL;
2394         int rc = 0;
2395         struct fw_caps_config_cmd caps;
2396         uint32_t mtype, moff, finicsum, cfcsum;
2397
2398         /*
2399          * Figure out what configuration file to use.  Pick the default config
2400          * file for the card if the user hasn't specified one explicitly.
2401          */
2402         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2403         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2404                 /* Card specific overrides go here. */
2405                 if (pci_get_device(sc->dev) == 0x440a)
2406                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2407                 if (is_fpga(sc))
2408                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2409         }
2410
2411         /*
2412          * We need to load another module if the profile is anything except
2413          * "default" or "flash".
2414          */
2415         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2416             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2417                 char s[32];
2418
2419                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2420                 cfg = firmware_get(s);
2421                 if (cfg == NULL) {
2422                         if (default_cfg != NULL) {
2423                                 device_printf(sc->dev,
2424                                     "unable to load module \"%s\" for "
2425                                     "configuration profile \"%s\", will use "
2426                                     "the default config file instead.\n",
2427                                     s, sc->cfg_file);
2428                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2429                                     "%s", DEFAULT_CF);
2430                         } else {
2431                                 device_printf(sc->dev,
2432                                     "unable to load module \"%s\" for "
2433                                     "configuration profile \"%s\", will use "
2434                                     "the config file on the card's flash "
2435                                     "instead.\n", s, sc->cfg_file);
2436                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2437                                     "%s", FLASH_CF);
2438                         }
2439                 }
2440         }
2441
2442         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2443             default_cfg == NULL) {
2444                 device_printf(sc->dev,
2445                     "default config file not available, will use the config "
2446                     "file on the card's flash instead.\n");
2447                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2448         }
2449
2450         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2451                 u_int cflen, i, n;
2452                 const uint32_t *cfdata;
2453                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2454
2455                 KASSERT(cfg != NULL || default_cfg != NULL,
2456                     ("%s: no config to upload", __func__));
2457
2458                 /*
2459                  * Ask the firmware where it wants us to upload the config file.
2460                  */
2461                 param = FW_PARAM_DEV(CF);
2462                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2463                 if (rc != 0) {
2464                         /* No support for config file?  Shouldn't happen. */
2465                         device_printf(sc->dev,
2466                             "failed to query config file location: %d.\n", rc);
2467                         goto done;
2468                 }
2469                 mtype = G_FW_PARAMS_PARAM_Y(val);
2470                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2471
2472                 /*
2473                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2474                  * useless stuffing/comments at the end of the config file so
2475                  * it's ok to simply throw away the last remaining bytes when
2476                  * the config file is not an exact multiple of 4.  This also
2477                  * helps with the validate_mt_off_len check.
2478                  */
2479                 if (cfg != NULL) {
2480                         cflen = cfg->datasize & ~3;
2481                         cfdata = cfg->data;
2482                 } else {
2483                         cflen = default_cfg->datasize & ~3;
2484                         cfdata = default_cfg->data;
2485                 }
2486
2487                 if (cflen > FLASH_CFG_MAX_SIZE) {
2488                         device_printf(sc->dev,
2489                             "config file too long (%d, max allowed is %d).  "
2490                             "Will try to use the config on the card, if any.\n",
2491                             cflen, FLASH_CFG_MAX_SIZE);
2492                         goto use_config_on_flash;
2493                 }
2494
2495                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2496                 if (rc != 0) {
2497                         device_printf(sc->dev,
2498                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2499                             "Will try to use the config on the card, if any.\n",
2500                             __func__, mtype, moff, cflen, rc);
2501                         goto use_config_on_flash;
2502                 }
2503
2504                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2505                 while (cflen) {
2506                         off = position_memwin(sc, 2, addr);
2507                         n = min(cflen, mw_aperture - off);
2508                         for (i = 0; i < n; i += 4)
2509                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2510                         cflen -= n;
2511                         addr += n;
2512                 }
2513         } else {
2514 use_config_on_flash:
2515                 mtype = FW_MEMTYPE_FLASH;
2516                 moff = t4_flash_cfg_addr(sc);
2517         }
2518
2519         bzero(&caps, sizeof(caps));
2520         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2521             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2522         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2523             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2524             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2525         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2526         if (rc != 0) {
2527                 device_printf(sc->dev,
2528                     "failed to pre-process config file: %d "
2529                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2530                 goto done;
2531         }
2532
2533         finicsum = be32toh(caps.finicsum);
2534         cfcsum = be32toh(caps.cfcsum);
2535         if (finicsum != cfcsum) {
2536                 device_printf(sc->dev,
2537                     "WARNING: config file checksum mismatch: %08x %08x\n",
2538                     finicsum, cfcsum);
2539         }
2540         sc->cfcsum = cfcsum;
2541
2542 #define LIMIT_CAPS(x) do { \
2543         caps.x &= htobe16(t4_##x##_allowed); \
2544 } while (0)
2545
2546         /*
2547          * Let the firmware know what features will (not) be used so it can tune
2548          * things accordingly.
2549          */
2550         LIMIT_CAPS(linkcaps);
2551         LIMIT_CAPS(niccaps);
2552         LIMIT_CAPS(toecaps);
2553         LIMIT_CAPS(rdmacaps);
2554         LIMIT_CAPS(iscsicaps);
2555         LIMIT_CAPS(fcoecaps);
2556 #undef LIMIT_CAPS
2557
2558         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2559             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2560         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2561         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2562         if (rc != 0) {
2563                 device_printf(sc->dev,
2564                     "failed to process config file: %d.\n", rc);
2565         }
2566 done:
2567         if (cfg != NULL)
2568                 firmware_put(cfg, FIRMWARE_UNLOAD);
2569         return (rc);
2570 }
2571
2572 /*
2573  * Retrieve parameters that are needed (or nice to have) very early.
2574  */
2575 static int
2576 get_params__pre_init(struct adapter *sc)
2577 {
2578         int rc;
2579         uint32_t param[2], val[2];
2580         struct fw_devlog_cmd cmd;
2581         struct devlog_params *dlog = &sc->params.devlog;
2582
2583         param[0] = FW_PARAM_DEV(PORTVEC);
2584         param[1] = FW_PARAM_DEV(CCLK);
2585         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2586         if (rc != 0) {
2587                 device_printf(sc->dev,
2588                     "failed to query parameters (pre_init): %d.\n", rc);
2589                 return (rc);
2590         }
2591
2592         sc->params.portvec = val[0];
2593         sc->params.nports = bitcount32(val[0]);
2594         sc->params.vpd.cclk = val[1];
2595
2596         /* Read device log parameters. */
2597         bzero(&cmd, sizeof(cmd));
2598         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2599             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2600         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2601         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2602         if (rc != 0) {
2603                 device_printf(sc->dev,
2604                     "failed to get devlog parameters: %d.\n", rc);
2605                 bzero(dlog, sizeof (*dlog));
2606                 rc = 0; /* devlog isn't critical for device operation */
2607         } else {
2608                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2609                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2610                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2611                 dlog->size = be32toh(cmd.memsize_devlog);
2612         }
2613
2614         return (rc);
2615 }
2616
2617 /*
2618  * Retrieve various parameters that are of interest to the driver.  The device
2619  * has been initialized by the firmware at this point.
2620  */
2621 static int
2622 get_params__post_init(struct adapter *sc)
2623 {
2624         int rc;
2625         uint32_t param[7], val[7];
2626         struct fw_caps_config_cmd caps;
2627
2628         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2629         param[1] = FW_PARAM_PFVF(EQ_START);
2630         param[2] = FW_PARAM_PFVF(FILTER_START);
2631         param[3] = FW_PARAM_PFVF(FILTER_END);
2632         param[4] = FW_PARAM_PFVF(L2T_START);
2633         param[5] = FW_PARAM_PFVF(L2T_END);
2634         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2635         if (rc != 0) {
2636                 device_printf(sc->dev,
2637                     "failed to query parameters (post_init): %d.\n", rc);
2638                 return (rc);
2639         }
2640
2641         sc->sge.iq_start = val[0];
2642         sc->sge.eq_start = val[1];
2643         sc->tids.ftid_base = val[2];
2644         sc->tids.nftids = val[3] - val[2] + 1;
2645         sc->params.ftid_min = val[2];
2646         sc->params.ftid_max = val[3];
2647         sc->vres.l2t.start = val[4];
2648         sc->vres.l2t.size = val[5] - val[4] + 1;
2649         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2650             ("%s: L2 table size (%u) larger than expected (%u)",
2651             __func__, sc->vres.l2t.size, L2T_SIZE));
2652
2653         /* get capabilites */
2654         bzero(&caps, sizeof(caps));
2655         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2656             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2657         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2658         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2659         if (rc != 0) {
2660                 device_printf(sc->dev,
2661                     "failed to get card capabilities: %d.\n", rc);
2662                 return (rc);
2663         }
2664
2665 #define READ_CAPS(x) do { \
2666         sc->x = htobe16(caps.x); \
2667 } while (0)
2668         READ_CAPS(linkcaps);
2669         READ_CAPS(niccaps);
2670         READ_CAPS(toecaps);
2671         READ_CAPS(rdmacaps);
2672         READ_CAPS(iscsicaps);
2673         READ_CAPS(fcoecaps);
2674
2675         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2676                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2677                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2678                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2679                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2680                 if (rc != 0) {
2681                         device_printf(sc->dev,
2682                             "failed to query NIC parameters: %d.\n", rc);
2683                         return (rc);
2684                 }
2685                 sc->tids.etid_base = val[0];
2686                 sc->params.etid_min = val[0];
2687                 sc->tids.netids = val[1] - val[0] + 1;
2688                 sc->params.netids = sc->tids.netids;
2689                 sc->params.eo_wr_cred = val[2];
2690                 sc->params.ethoffload = 1;
2691         }
2692
2693         if (sc->toecaps) {
2694                 /* query offload-related parameters */
2695                 param[0] = FW_PARAM_DEV(NTID);
2696                 param[1] = FW_PARAM_PFVF(SERVER_START);
2697                 param[2] = FW_PARAM_PFVF(SERVER_END);
2698                 param[3] = FW_PARAM_PFVF(TDDP_START);
2699                 param[4] = FW_PARAM_PFVF(TDDP_END);
2700                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2701                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2702                 if (rc != 0) {
2703                         device_printf(sc->dev,
2704                             "failed to query TOE parameters: %d.\n", rc);
2705                         return (rc);
2706                 }
2707                 sc->tids.ntids = val[0];
2708                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2709                 sc->tids.stid_base = val[1];
2710                 sc->tids.nstids = val[2] - val[1] + 1;
2711                 sc->vres.ddp.start = val[3];
2712                 sc->vres.ddp.size = val[4] - val[3] + 1;
2713                 sc->params.ofldq_wr_cred = val[5];
2714                 sc->params.offload = 1;
2715         }
2716         if (sc->rdmacaps) {
2717                 param[0] = FW_PARAM_PFVF(STAG_START);
2718                 param[1] = FW_PARAM_PFVF(STAG_END);
2719                 param[2] = FW_PARAM_PFVF(RQ_START);
2720                 param[3] = FW_PARAM_PFVF(RQ_END);
2721                 param[4] = FW_PARAM_PFVF(PBL_START);
2722                 param[5] = FW_PARAM_PFVF(PBL_END);
2723                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2724                 if (rc != 0) {
2725                         device_printf(sc->dev,
2726                             "failed to query RDMA parameters(1): %d.\n", rc);
2727                         return (rc);
2728                 }
2729                 sc->vres.stag.start = val[0];
2730                 sc->vres.stag.size = val[1] - val[0] + 1;
2731                 sc->vres.rq.start = val[2];
2732                 sc->vres.rq.size = val[3] - val[2] + 1;
2733                 sc->vres.pbl.start = val[4];
2734                 sc->vres.pbl.size = val[5] - val[4] + 1;
2735
2736                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2737                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2738                 param[2] = FW_PARAM_PFVF(CQ_START);
2739                 param[3] = FW_PARAM_PFVF(CQ_END);
2740                 param[4] = FW_PARAM_PFVF(OCQ_START);
2741                 param[5] = FW_PARAM_PFVF(OCQ_END);
2742                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2743                 if (rc != 0) {
2744                         device_printf(sc->dev,
2745                             "failed to query RDMA parameters(2): %d.\n", rc);
2746                         return (rc);
2747                 }
2748                 sc->vres.qp.start = val[0];
2749                 sc->vres.qp.size = val[1] - val[0] + 1;
2750                 sc->vres.cq.start = val[2];
2751                 sc->vres.cq.size = val[3] - val[2] + 1;
2752                 sc->vres.ocq.start = val[4];
2753                 sc->vres.ocq.size = val[5] - val[4] + 1;
2754         }
2755         if (sc->iscsicaps) {
2756                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2757                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2758                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2759                 if (rc != 0) {
2760                         device_printf(sc->dev,
2761                             "failed to query iSCSI parameters: %d.\n", rc);
2762                         return (rc);
2763                 }
2764                 sc->vres.iscsi.start = val[0];
2765                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2766         }
2767
2768         /*
2769          * We've got the params we wanted to query via the firmware.  Now grab
2770          * some others directly from the chip.
2771          */
2772         rc = t4_read_chip_settings(sc);
2773
2774         return (rc);
2775 }
2776
2777 static int
2778 set_params__post_init(struct adapter *sc)
2779 {
2780         uint32_t param, val;
2781
2782         /* ask for encapsulated CPLs */
2783         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2784         val = 1;
2785         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2786
2787         return (0);
2788 }
2789
2790 #undef FW_PARAM_PFVF
2791 #undef FW_PARAM_DEV
2792
2793 static void
2794 t4_set_desc(struct adapter *sc)
2795 {
2796         char buf[128];
2797         struct adapter_params *p = &sc->params;
2798
2799         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2800             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2801             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2802
2803         device_set_desc_copy(sc->dev, buf);
2804 }
2805
2806 static void
2807 build_medialist(struct port_info *pi, struct ifmedia *media)
2808 {
2809         int data, m;
2810
2811         PORT_LOCK(pi);
2812
2813         ifmedia_removeall(media);
2814
2815         m = IFM_ETHER | IFM_FDX;
2816         data = (pi->port_type << 8) | pi->mod_type;
2817
2818         switch(pi->port_type) {
2819         case FW_PORT_TYPE_BT_XFI:
2820                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2821                 break;
2822
2823         case FW_PORT_TYPE_BT_XAUI:
2824                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2825                 /* fall through */
2826
2827         case FW_PORT_TYPE_BT_SGMII:
2828                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2829                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2830                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2831                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2832                 break;
2833
2834         case FW_PORT_TYPE_CX4:
2835                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2836                 ifmedia_set(media, m | IFM_10G_CX4);
2837                 break;
2838
2839         case FW_PORT_TYPE_QSFP_10G:
2840         case FW_PORT_TYPE_SFP:
2841         case FW_PORT_TYPE_FIBER_XFI:
2842         case FW_PORT_TYPE_FIBER_XAUI:
2843                 switch (pi->mod_type) {
2844
2845                 case FW_PORT_MOD_TYPE_LR:
2846                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2847                         ifmedia_set(media, m | IFM_10G_LR);
2848                         break;
2849
2850                 case FW_PORT_MOD_TYPE_SR:
2851                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2852                         ifmedia_set(media, m | IFM_10G_SR);
2853                         break;
2854
2855                 case FW_PORT_MOD_TYPE_LRM:
2856                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2857                         ifmedia_set(media, m | IFM_10G_LRM);
2858                         break;
2859
2860                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2861                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2862                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2863                         ifmedia_set(media, m | IFM_10G_TWINAX);
2864                         break;
2865
2866                 case FW_PORT_MOD_TYPE_NONE:
2867                         m &= ~IFM_FDX;
2868                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2869                         ifmedia_set(media, m | IFM_NONE);
2870                         break;
2871
2872                 case FW_PORT_MOD_TYPE_NA:
2873                 case FW_PORT_MOD_TYPE_ER:
2874                 default:
2875                         device_printf(pi->dev,
2876                             "unknown port_type (%d), mod_type (%d)\n",
2877                             pi->port_type, pi->mod_type);
2878                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2879                         ifmedia_set(media, m | IFM_UNKNOWN);
2880                         break;
2881                 }
2882                 break;
2883
2884         case FW_PORT_TYPE_QSFP:
2885                 switch (pi->mod_type) {
2886
2887                 case FW_PORT_MOD_TYPE_LR:
2888                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2889                         ifmedia_set(media, m | IFM_40G_LR4);
2890                         break;
2891
2892                 case FW_PORT_MOD_TYPE_SR:
2893                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2894                         ifmedia_set(media, m | IFM_40G_SR4);
2895                         break;
2896
2897                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2898                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2899                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2900                         ifmedia_set(media, m | IFM_40G_CR4);
2901                         break;
2902
2903                 case FW_PORT_MOD_TYPE_NONE:
2904                         m &= ~IFM_FDX;
2905                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2906                         ifmedia_set(media, m | IFM_NONE);
2907                         break;
2908
2909                 default:
2910                         device_printf(pi->dev,
2911                             "unknown port_type (%d), mod_type (%d)\n",
2912                             pi->port_type, pi->mod_type);
2913                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2914                         ifmedia_set(media, m | IFM_UNKNOWN);
2915                         break;
2916                 }
2917                 break;
2918
2919         default:
2920                 device_printf(pi->dev,
2921                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2922                     pi->mod_type);
2923                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2924                 ifmedia_set(media, m | IFM_UNKNOWN);
2925                 break;
2926         }
2927
2928         PORT_UNLOCK(pi);
2929 }
2930
2931 #define FW_MAC_EXACT_CHUNK      7
2932
2933 /*
2934  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2935  * indicates which parameters should be programmed (the rest are left alone).
2936  */
2937 int
2938 update_mac_settings(struct ifnet *ifp, int flags)
2939 {
2940         int rc = 0;
2941         struct port_info *pi = ifp->if_softc;
2942         struct adapter *sc = pi->adapter;
2943         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2944         uint16_t viid = 0xffff;
2945         int16_t *xact_addr_filt = NULL;
2946
2947         ASSERT_SYNCHRONIZED_OP(sc);
2948         KASSERT(flags, ("%s: not told what to update.", __func__));
2949
2950         if (ifp == pi->ifp) {
2951                 viid = pi->viid;
2952                 xact_addr_filt = &pi->xact_addr_filt;
2953         }
2954 #ifdef DEV_NETMAP
2955         else if (ifp == pi->nm_ifp) {
2956                 viid = pi->nm_viid;
2957                 xact_addr_filt = &pi->nm_xact_addr_filt;
2958         }
2959 #endif
2960         if (flags & XGMAC_MTU)
2961                 mtu = ifp->if_mtu;
2962
2963         if (flags & XGMAC_PROMISC)
2964                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2965
2966         if (flags & XGMAC_ALLMULTI)
2967                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2968
2969         if (flags & XGMAC_VLANEX)
2970                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2971
2972         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2973                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2974                     1, vlanex, false);
2975                 if (rc) {
2976                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2977                             rc);
2978                         return (rc);
2979                 }
2980         }
2981
2982         if (flags & XGMAC_UCADDR) {
2983                 uint8_t ucaddr[ETHER_ADDR_LEN];
2984
2985                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2986                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2987                     true, true);
2988                 if (rc < 0) {
2989                         rc = -rc;
2990                         if_printf(ifp, "change_mac failed: %d\n", rc);
2991                         return (rc);
2992                 } else {
2993                         *xact_addr_filt = rc;
2994                         rc = 0;
2995                 }
2996         }
2997
2998         if (flags & XGMAC_MCADDRS) {
2999                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3000                 int del = 1;
3001                 uint64_t hash = 0;
3002                 struct ifmultiaddr *ifma;
3003                 int i = 0, j;
3004
3005                 if_maddr_rlock(ifp);
3006                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3007                         if (ifma->ifma_addr->sa_family != AF_LINK)
3008                                 continue;
3009                         mcaddr[i++] =
3010                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3011
3012                         if (i == FW_MAC_EXACT_CHUNK) {
3013                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3014                                     i, mcaddr, NULL, &hash, 0);
3015                                 if (rc < 0) {
3016                                         rc = -rc;
3017                                         for (j = 0; j < i; j++) {
3018                                                 if_printf(ifp,
3019                                                     "failed to add mc address"
3020                                                     " %02x:%02x:%02x:"
3021                                                     "%02x:%02x:%02x rc=%d\n",
3022                                                     mcaddr[j][0], mcaddr[j][1],
3023                                                     mcaddr[j][2], mcaddr[j][3],
3024                                                     mcaddr[j][4], mcaddr[j][5],
3025                                                     rc);
3026                                         }
3027                                         goto mcfail;
3028                                 }
3029                                 del = 0;
3030                                 i = 0;
3031                         }
3032                 }
3033                 if (i > 0) {
3034                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3035                             mcaddr, NULL, &hash, 0);
3036                         if (rc < 0) {
3037                                 rc = -rc;
3038                                 for (j = 0; j < i; j++) {
3039                                         if_printf(ifp,
3040                                             "failed to add mc address"
3041                                             " %02x:%02x:%02x:"
3042                                             "%02x:%02x:%02x rc=%d\n",
3043                                             mcaddr[j][0], mcaddr[j][1],
3044                                             mcaddr[j][2], mcaddr[j][3],
3045                                             mcaddr[j][4], mcaddr[j][5],
3046                                             rc);
3047                                 }
3048                                 goto mcfail;
3049                         }
3050                 }
3051
3052                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3053                 if (rc != 0)
3054                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3055 mcfail:
3056                 if_maddr_runlock(ifp);
3057         }
3058
3059         return (rc);
3060 }
3061
3062 int
3063 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3064     char *wmesg)
3065 {
3066         int rc, pri;
3067
3068 #ifdef WITNESS
3069         /* the caller thinks it's ok to sleep, but is it really? */
3070         if (flags & SLEEP_OK)
3071                 pause("t4slptst", 1);
3072 #endif
3073
3074         if (INTR_OK)
3075                 pri = PCATCH;
3076         else
3077                 pri = 0;
3078
3079         ADAPTER_LOCK(sc);
3080         for (;;) {
3081
3082                 if (pi && IS_DOOMED(pi)) {
3083                         rc = ENXIO;
3084                         goto done;
3085                 }
3086
3087                 if (!IS_BUSY(sc)) {
3088                         rc = 0;
3089                         break;
3090                 }
3091
3092                 if (!(flags & SLEEP_OK)) {
3093                         rc = EBUSY;
3094                         goto done;
3095                 }
3096
3097                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3098                         rc = EINTR;
3099                         goto done;
3100                 }
3101         }
3102
3103         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3104         SET_BUSY(sc);
3105 #ifdef INVARIANTS
3106         sc->last_op = wmesg;
3107         sc->last_op_thr = curthread;
3108 #endif
3109
3110 done:
3111         if (!(flags & HOLD_LOCK) || rc)
3112                 ADAPTER_UNLOCK(sc);
3113
3114         return (rc);
3115 }
3116
3117 void
3118 end_synchronized_op(struct adapter *sc, int flags)
3119 {
3120
3121         if (flags & LOCK_HELD)
3122                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3123         else
3124                 ADAPTER_LOCK(sc);
3125
3126         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3127         CLR_BUSY(sc);
3128         wakeup(&sc->flags);
3129         ADAPTER_UNLOCK(sc);
3130 }
3131
3132 static int
3133 cxgbe_init_synchronized(struct port_info *pi)
3134 {
3135         struct adapter *sc = pi->adapter;
3136         struct ifnet *ifp = pi->ifp;
3137         int rc = 0;
3138
3139         ASSERT_SYNCHRONIZED_OP(sc);
3140
3141         if (isset(&sc->open_device_map, pi->port_id)) {
3142                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3143                     ("mismatch between open_device_map and if_drv_flags"));
3144                 return (0);     /* already running */
3145         }
3146
3147         if (!(sc->flags & FULL_INIT_DONE) &&
3148             ((rc = adapter_full_init(sc)) != 0))
3149                 return (rc);    /* error message displayed already */
3150
3151         if (!(pi->flags & PORT_INIT_DONE) &&
3152             ((rc = port_full_init(pi)) != 0))
3153                 return (rc); /* error message displayed already */
3154
3155         rc = update_mac_settings(ifp, XGMAC_ALL);
3156         if (rc)
3157                 goto done;      /* error message displayed already */
3158
3159         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3160         if (rc != 0) {
3161                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3162                 goto done;
3163         }
3164
3165         /*
3166          * The first iq of the first port to come up is used for tracing.
3167          */
3168         if (sc->traceq < 0) {
3169                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3170                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3171                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3172                     V_QUEUENUMBER(sc->traceq));
3173                 pi->flags |= HAS_TRACEQ;
3174         }
3175
3176         /* all ok */
3177         setbit(&sc->open_device_map, pi->port_id);
3178         PORT_LOCK(pi);
3179         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3180         PORT_UNLOCK(pi);
3181
3182         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3183 done:
3184         if (rc != 0)
3185                 cxgbe_uninit_synchronized(pi);
3186
3187         return (rc);
3188 }
3189
3190 /*
3191  * Idempotent.
3192  */
3193 static int
3194 cxgbe_uninit_synchronized(struct port_info *pi)
3195 {
3196         struct adapter *sc = pi->adapter;
3197         struct ifnet *ifp = pi->ifp;
3198         int rc;
3199
3200         ASSERT_SYNCHRONIZED_OP(sc);
3201
3202         /*
3203          * Disable the VI so that all its data in either direction is discarded
3204          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3205          * tick) intact as the TP can deliver negative advice or data that it's
3206          * holding in its RAM (for an offloaded connection) even after the VI is
3207          * disabled.
3208          */
3209         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3210         if (rc) {
3211                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3212                 return (rc);
3213         }
3214
3215         clrbit(&sc->open_device_map, pi->port_id);
3216         PORT_LOCK(pi);
3217         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3218         PORT_UNLOCK(pi);
3219
3220         pi->link_cfg.link_ok = 0;
3221         pi->link_cfg.speed = 0;
3222         pi->linkdnrc = -1;
3223         t4_os_link_changed(sc, pi->port_id, 0, -1);
3224
3225         return (0);
3226 }
3227
3228 /*
3229  * It is ok for this function to fail midway and return right away.  t4_detach
3230  * will walk the entire sc->irq list and clean up whatever is valid.
3231  */
3232 static int
3233 setup_intr_handlers(struct adapter *sc)
3234 {
3235         int rc, rid, p, q;
3236         char s[8];
3237         struct irq *irq;
3238         struct port_info *pi;
3239         struct sge_rxq *rxq;
3240 #ifdef TCP_OFFLOAD
3241         struct sge_ofld_rxq *ofld_rxq;
3242 #endif
3243 #ifdef DEV_NETMAP
3244         struct sge_nm_rxq *nm_rxq;
3245 #endif
3246
3247         /*
3248          * Setup interrupts.
3249          */
3250         irq = &sc->irq[0];
3251         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3252         if (sc->intr_count == 1)
3253                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3254
3255         /* Multiple interrupts. */
3256         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3257             ("%s: too few intr.", __func__));
3258
3259         /* The first one is always error intr */
3260         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3261         if (rc != 0)
3262                 return (rc);
3263         irq++;
3264         rid++;
3265
3266         /* The second one is always the firmware event queue */
3267         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3268         if (rc != 0)
3269                 return (rc);
3270         irq++;
3271         rid++;
3272
3273         for_each_port(sc, p) {
3274                 pi = sc->port[p];
3275
3276                 if (pi->flags & INTR_RXQ) {
3277                         for_each_rxq(pi, q, rxq) {
3278                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3279                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3280                                     s);
3281                                 if (rc != 0)
3282                                         return (rc);
3283                                 irq++;
3284                                 rid++;
3285                         }
3286                 }
3287 #ifdef TCP_OFFLOAD
3288                 if (pi->flags & INTR_OFLD_RXQ) {
3289                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3290                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3291                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3292                                     ofld_rxq, s);
3293                                 if (rc != 0)
3294                                         return (rc);
3295                                 irq++;
3296                                 rid++;
3297                         }
3298                 }
3299 #endif
3300 #ifdef DEV_NETMAP
3301                 if (pi->flags & INTR_NM_RXQ) {
3302                         for_each_nm_rxq(pi, q, nm_rxq) {
3303                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3304                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3305                                     nm_rxq, s);
3306                                 if (rc != 0)
3307                                         return (rc);
3308                                 irq++;
3309                                 rid++;
3310                         }
3311                 }
3312 #endif
3313         }
3314         MPASS(irq == &sc->irq[sc->intr_count]);
3315
3316         return (0);
3317 }
3318
3319 int
3320 adapter_full_init(struct adapter *sc)
3321 {
3322         int rc, i;
3323
3324         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3325         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3326             ("%s: FULL_INIT_DONE already", __func__));
3327
3328         /*
3329          * queues that belong to the adapter (not any particular port).
3330          */
3331         rc = t4_setup_adapter_queues(sc);
3332         if (rc != 0)
3333                 goto done;
3334
3335         for (i = 0; i < nitems(sc->tq); i++) {
3336                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3337                     taskqueue_thread_enqueue, &sc->tq[i]);
3338                 if (sc->tq[i] == NULL) {
3339                         device_printf(sc->dev,
3340                             "failed to allocate task queue %d\n", i);
3341                         rc = ENOMEM;
3342                         goto done;
3343                 }
3344                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3345                     device_get_nameunit(sc->dev), i);
3346         }
3347
3348         t4_intr_enable(sc);
3349         sc->flags |= FULL_INIT_DONE;
3350 done:
3351         if (rc != 0)
3352                 adapter_full_uninit(sc);
3353
3354         return (rc);
3355 }
3356
3357 int
3358 adapter_full_uninit(struct adapter *sc)
3359 {
3360         int i;
3361
3362         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3363
3364         t4_teardown_adapter_queues(sc);
3365
3366         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3367                 taskqueue_free(sc->tq[i]);
3368                 sc->tq[i] = NULL;
3369         }
3370
3371         sc->flags &= ~FULL_INIT_DONE;
3372
3373         return (0);
3374 }
3375
3376 int
3377 port_full_init(struct port_info *pi)
3378 {
3379         struct adapter *sc = pi->adapter;
3380         struct ifnet *ifp = pi->ifp;
3381         uint16_t *rss;
3382         struct sge_rxq *rxq;
3383         int rc, i, j;
3384
3385         ASSERT_SYNCHRONIZED_OP(sc);
3386         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3387             ("%s: PORT_INIT_DONE already", __func__));
3388
3389         sysctl_ctx_init(&pi->ctx);
3390         pi->flags |= PORT_SYSCTL_CTX;
3391
3392         /*
3393          * Allocate tx/rx/fl queues for this port.
3394          */
3395         rc = t4_setup_port_queues(pi);
3396         if (rc != 0)
3397                 goto done;      /* error message displayed already */
3398
3399         /*
3400          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3401          */
3402         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3403         for (i = 0; i < pi->rss_size;) {
3404                 for_each_rxq(pi, j, rxq) {
3405                         rss[i++] = rxq->iq.abs_id;
3406                         if (i == pi->rss_size)
3407                                 break;
3408                 }
3409         }
3410
3411         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3412             pi->rss_size);
3413         if (rc != 0) {
3414                 if_printf(ifp, "rss_config failed: %d\n", rc);
3415                 goto done;
3416         }
3417
3418         pi->rss = rss;
3419         pi->flags |= PORT_INIT_DONE;
3420 done:
3421         if (rc != 0)
3422                 port_full_uninit(pi);
3423
3424         return (rc);
3425 }
3426
3427 /*
3428  * Idempotent.
3429  */
3430 int
3431 port_full_uninit(struct port_info *pi)
3432 {
3433         struct adapter *sc = pi->adapter;
3434         int i;
3435         struct sge_rxq *rxq;
3436         struct sge_txq *txq;
3437 #ifdef TCP_OFFLOAD
3438         struct sge_ofld_rxq *ofld_rxq;
3439         struct sge_wrq *ofld_txq;
3440 #endif
3441
3442         if (pi->flags & PORT_INIT_DONE) {
3443
3444                 /* Need to quiesce queues.  XXX: ctrl queues? */
3445
3446                 for_each_txq(pi, i, txq) {
3447                         quiesce_eq(sc, &txq->eq);
3448                 }
3449
3450 #ifdef TCP_OFFLOAD
3451                 for_each_ofld_txq(pi, i, ofld_txq) {
3452                         quiesce_eq(sc, &ofld_txq->eq);
3453                 }
3454 #endif
3455
3456                 for_each_rxq(pi, i, rxq) {
3457                         quiesce_iq(sc, &rxq->iq);
3458                         quiesce_fl(sc, &rxq->fl);
3459                 }
3460
3461 #ifdef TCP_OFFLOAD
3462                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3463                         quiesce_iq(sc, &ofld_rxq->iq);
3464                         quiesce_fl(sc, &ofld_rxq->fl);
3465                 }
3466 #endif
3467                 free(pi->rss, M_CXGBE);
3468         }
3469
3470         t4_teardown_port_queues(pi);
3471         pi->flags &= ~PORT_INIT_DONE;
3472
3473         return (0);
3474 }
3475
3476 static void
3477 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3478 {
3479         EQ_LOCK(eq);
3480         eq->flags |= EQ_DOOMED;
3481
3482         /*
3483          * Wait for the response to a credit flush if one's
3484          * pending.
3485          */
3486         while (eq->flags & EQ_CRFLUSHED)
3487                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3488         EQ_UNLOCK(eq);
3489
3490         callout_drain(&eq->tx_callout); /* XXX: iffy */
3491         pause("callout", 10);           /* Still iffy */
3492
3493         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3494 }
3495
3496 static void
3497 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3498 {
3499         (void) sc;      /* unused */
3500
3501         /* Synchronize with the interrupt handler */
3502         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3503                 pause("iqfree", 1);
3504 }
3505
3506 static void
3507 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3508 {
3509         mtx_lock(&sc->sfl_lock);
3510         FL_LOCK(fl);
3511         fl->flags |= FL_DOOMED;
3512         FL_UNLOCK(fl);
3513         mtx_unlock(&sc->sfl_lock);
3514
3515         callout_drain(&sc->sfl_callout);
3516         KASSERT((fl->flags & FL_STARVING) == 0,
3517             ("%s: still starving", __func__));
3518 }
3519
3520 static int
3521 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3522     driver_intr_t *handler, void *arg, char *name)
3523 {
3524         int rc;
3525
3526         irq->rid = rid;
3527         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3528             RF_SHAREABLE | RF_ACTIVE);
3529         if (irq->res == NULL) {
3530                 device_printf(sc->dev,
3531                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3532                 return (ENOMEM);
3533         }
3534
3535         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3536             NULL, handler, arg, &irq->tag);
3537         if (rc != 0) {
3538                 device_printf(sc->dev,
3539                     "failed to setup interrupt for rid %d, name %s: %d\n",
3540                     rid, name, rc);
3541         } else if (name)
3542                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3543
3544         return (rc);
3545 }
3546
3547 static int
3548 t4_free_irq(struct adapter *sc, struct irq *irq)
3549 {
3550         if (irq->tag)
3551                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3552         if (irq->res)
3553                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3554
3555         bzero(irq, sizeof(*irq));
3556
3557         return (0);
3558 }
3559
3560 static void
3561 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3562     unsigned int end)
3563 {
3564         uint32_t *p = (uint32_t *)(buf + start);
3565
3566         for ( ; start <= end; start += sizeof(uint32_t))
3567                 *p++ = t4_read_reg(sc, start);
3568 }
3569
3570 static void
3571 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3572 {
3573         int i, n;
3574         const unsigned int *reg_ranges;
3575         static const unsigned int t4_reg_ranges[] = {
3576                 0x1008, 0x1108,
3577                 0x1180, 0x11b4,
3578                 0x11fc, 0x123c,
3579                 0x1300, 0x173c,
3580                 0x1800, 0x18fc,
3581                 0x3000, 0x30d8,
3582                 0x30e0, 0x5924,
3583                 0x5960, 0x59d4,
3584                 0x5a00, 0x5af8,
3585                 0x6000, 0x6098,
3586                 0x6100, 0x6150,
3587                 0x6200, 0x6208,
3588                 0x6240, 0x6248,
3589                 0x6280, 0x6338,
3590                 0x6370, 0x638c,
3591                 0x6400, 0x643c,
3592                 0x6500, 0x6524,
3593                 0x6a00, 0x6a38,
3594                 0x6a60, 0x6a78,
3595                 0x6b00, 0x6b84,
3596                 0x6bf0, 0x6c84,
3597                 0x6cf0, 0x6d84,
3598                 0x6df0, 0x6e84,
3599                 0x6ef0, 0x6f84,
3600                 0x6ff0, 0x7084,
3601                 0x70f0, 0x7184,
3602                 0x71f0, 0x7284,
3603                 0x72f0, 0x7384,
3604                 0x73f0, 0x7450,
3605                 0x7500, 0x7530,
3606                 0x7600, 0x761c,
3607                 0x7680, 0x76cc,
3608                 0x7700, 0x7798,
3609                 0x77c0, 0x77fc,
3610                 0x7900, 0x79fc,
3611                 0x7b00, 0x7c38,
3612                 0x7d00, 0x7efc,
3613                 0x8dc0, 0x8e1c,
3614                 0x8e30, 0x8e78,
3615                 0x8ea0, 0x8f6c,
3616                 0x8fc0, 0x9074,
3617                 0x90fc, 0x90fc,
3618                 0x9400, 0x9458,
3619                 0x9600, 0x96bc,
3620                 0x9800, 0x9808,
3621                 0x9820, 0x983c,
3622                 0x9850, 0x9864,
3623                 0x9c00, 0x9c6c,
3624                 0x9c80, 0x9cec,
3625                 0x9d00, 0x9d6c,
3626                 0x9d80, 0x9dec,
3627                 0x9e00, 0x9e6c,
3628                 0x9e80, 0x9eec,
3629                 0x9f00, 0x9f6c,
3630                 0x9f80, 0x9fec,
3631                 0xd004, 0xd03c,
3632                 0xdfc0, 0xdfe0,
3633                 0xe000, 0xea7c,
3634                 0xf000, 0x11110,
3635                 0x11118, 0x11190,
3636                 0x19040, 0x1906c,
3637                 0x19078, 0x19080,
3638                 0x1908c, 0x19124,
3639                 0x19150, 0x191b0,
3640                 0x191d0, 0x191e8,
3641                 0x19238, 0x1924c,
3642                 0x193f8, 0x19474,
3643                 0x19490, 0x194f8,
3644                 0x19800, 0x19f30,
3645                 0x1a000, 0x1a06c,
3646                 0x1a0b0, 0x1a120,
3647                 0x1a128, 0x1a138,
3648                 0x1a190, 0x1a1c4,
3649                 0x1a1fc, 0x1a1fc,
3650                 0x1e040, 0x1e04c,
3651                 0x1e284, 0x1e28c,
3652                 0x1e2c0, 0x1e2c0,
3653                 0x1e2e0, 0x1e2e0,
3654                 0x1e300, 0x1e384,
3655                 0x1e3c0, 0x1e3c8,
3656                 0x1e440, 0x1e44c,
3657                 0x1e684, 0x1e68c,
3658                 0x1e6c0, 0x1e6c0,
3659                 0x1e6e0, 0x1e6e0,
3660                 0x1e700, 0x1e784,
3661                 0x1e7c0, 0x1e7c8,
3662                 0x1e840, 0x1e84c,
3663                 0x1ea84, 0x1ea8c,
3664                 0x1eac0, 0x1eac0,
3665                 0x1eae0, 0x1eae0,
3666                 0x1eb00, 0x1eb84,
3667                 0x1ebc0, 0x1ebc8,
3668                 0x1ec40, 0x1ec4c,
3669                 0x1ee84, 0x1ee8c,
3670                 0x1eec0, 0x1eec0,
3671                 0x1eee0, 0x1eee0,
3672                 0x1ef00, 0x1ef84,
3673                 0x1efc0, 0x1efc8,
3674                 0x1f040, 0x1f04c,
3675                 0x1f284, 0x1f28c,
3676                 0x1f2c0, 0x1f2c0,
3677                 0x1f2e0, 0x1f2e0,
3678                 0x1f300, 0x1f384,
3679                 0x1f3c0, 0x1f3c8,
3680                 0x1f440, 0x1f44c,
3681                 0x1f684, 0x1f68c,
3682                 0x1f6c0, 0x1f6c0,
3683                 0x1f6e0, 0x1f6e0,
3684                 0x1f700, 0x1f784,
3685                 0x1f7c0, 0x1f7c8,
3686                 0x1f840, 0x1f84c,
3687                 0x1fa84, 0x1fa8c,
3688                 0x1fac0, 0x1fac0,
3689                 0x1fae0, 0x1fae0,
3690                 0x1fb00, 0x1fb84,
3691                 0x1fbc0, 0x1fbc8,
3692                 0x1fc40, 0x1fc4c,
3693                 0x1fe84, 0x1fe8c,
3694                 0x1fec0, 0x1fec0,
3695                 0x1fee0, 0x1fee0,
3696                 0x1ff00, 0x1ff84,
3697                 0x1ffc0, 0x1ffc8,
3698                 0x20000, 0x2002c,
3699                 0x20100, 0x2013c,
3700                 0x20190, 0x201c8,
3701                 0x20200, 0x20318,
3702                 0x20400, 0x20528,
3703                 0x20540, 0x20614,
3704                 0x21000, 0x21040,
3705                 0x2104c, 0x21060,
3706                 0x210c0, 0x210ec,
3707                 0x21200, 0x21268,
3708                 0x21270, 0x21284,
3709                 0x212fc, 0x21388,
3710                 0x21400, 0x21404,
3711                 0x21500, 0x21518,
3712                 0x2152c, 0x2153c,
3713                 0x21550, 0x21554,
3714                 0x21600, 0x21600,
3715                 0x21608, 0x21628,
3716                 0x21630, 0x2163c,
3717                 0x21700, 0x2171c,
3718                 0x21780, 0x2178c,
3719                 0x21800, 0x21c38,
3720                 0x21c80, 0x21d7c,
3721                 0x21e00, 0x21e04,
3722                 0x22000, 0x2202c,
3723                 0x22100, 0x2213c,
3724                 0x22190, 0x221c8,
3725                 0x22200, 0x22318,
3726                 0x22400, 0x22528,
3727                 0x22540, 0x22614,
3728                 0x23000, 0x23040,
3729                 0x2304c, 0x23060,
3730                 0x230c0, 0x230ec,
3731                 0x23200, 0x23268,
3732                 0x23270, 0x23284,
3733                 0x232fc, 0x23388,
3734                 0x23400, 0x23404,
3735                 0x23500, 0x23518,
3736                 0x2352c, 0x2353c,
3737                 0x23550, 0x23554,
3738                 0x23600, 0x23600,
3739                 0x23608, 0x23628,
3740                 0x23630, 0x2363c,
3741                 0x23700, 0x2371c,
3742                 0x23780, 0x2378c,
3743                 0x23800, 0x23c38,
3744                 0x23c80, 0x23d7c,
3745                 0x23e00, 0x23e04,
3746                 0x24000, 0x2402c,
3747                 0x24100, 0x2413c,
3748                 0x24190, 0x241c8,
3749                 0x24200, 0x24318,
3750                 0x24400, 0x24528,
3751                 0x24540, 0x24614,
3752                 0x25000, 0x25040,
3753                 0x2504c, 0x25060,
3754                 0x250c0, 0x250ec,
3755                 0x25200, 0x25268,
3756                 0x25270, 0x25284,
3757                 0x252fc, 0x25388,
3758                 0x25400, 0x25404,
3759                 0x25500, 0x25518,
3760                 0x2552c, 0x2553c,
3761                 0x25550, 0x25554,
3762                 0x25600, 0x25600,
3763                 0x25608, 0x25628,
3764                 0x25630, 0x2563c,
3765                 0x25700, 0x2571c,
3766                 0x25780, 0x2578c,
3767                 0x25800, 0x25c38,
3768                 0x25c80, 0x25d7c,
3769                 0x25e00, 0x25e04,
3770                 0x26000, 0x2602c,
3771                 0x26100, 0x2613c,
3772                 0x26190, 0x261c8,
3773                 0x26200, 0x26318,
3774                 0x26400, 0x26528,
3775                 0x26540, 0x26614,
3776                 0x27000, 0x27040,
3777                 0x2704c, 0x27060,
3778                 0x270c0, 0x270ec,
3779                 0x27200, 0x27268,
3780                 0x27270, 0x27284,
3781                 0x272fc, 0x27388,
3782                 0x27400, 0x27404,
3783                 0x27500, 0x27518,
3784                 0x2752c, 0x2753c,
3785                 0x27550, 0x27554,
3786                 0x27600, 0x27600,
3787                 0x27608, 0x27628,
3788                 0x27630, 0x2763c,
3789                 0x27700, 0x2771c,
3790                 0x27780, 0x2778c,
3791                 0x27800, 0x27c38,
3792                 0x27c80, 0x27d7c,
3793                 0x27e00, 0x27e04
3794         };
3795         static const unsigned int t5_reg_ranges[] = {
3796                 0x1008, 0x1148,
3797                 0x1180, 0x11b4,
3798                 0x11fc, 0x123c,
3799                 0x1280, 0x173c,
3800                 0x1800, 0x18fc,
3801                 0x3000, 0x3028,
3802                 0x3060, 0x30d8,
3803                 0x30e0, 0x30fc,
3804                 0x3140, 0x357c,
3805                 0x35a8, 0x35cc,
3806                 0x35ec, 0x35ec,
3807                 0x3600, 0x5624,
3808                 0x56cc, 0x575c,
3809                 0x580c, 0x5814,
3810                 0x5890, 0x58bc,
3811                 0x5940, 0x59dc,
3812                 0x59fc, 0x5a18,
3813                 0x5a60, 0x5a9c,
3814                 0x5b94, 0x5bfc,
3815                 0x6000, 0x6040,
3816                 0x6058, 0x614c,
3817                 0x7700, 0x7798,
3818                 0x77c0, 0x78fc,
3819                 0x7b00, 0x7c54,
3820                 0x7d00, 0x7efc,
3821                 0x8dc0, 0x8de0,
3822                 0x8df8, 0x8e84,
3823                 0x8ea0, 0x8f84,
3824                 0x8fc0, 0x90f8,
3825                 0x9400, 0x9470,
3826                 0x9600, 0x96f4,
3827                 0x9800, 0x9808,
3828                 0x9820, 0x983c,
3829                 0x9850, 0x9864,
3830                 0x9c00, 0x9c6c,
3831                 0x9c80, 0x9cec,
3832                 0x9d00, 0x9d6c,
3833                 0x9d80, 0x9dec,
3834                 0x9e00, 0x9e6c,
3835                 0x9e80, 0x9eec,
3836                 0x9f00, 0x9f6c,
3837                 0x9f80, 0xa020,
3838                 0xd004, 0xd03c,
3839                 0xdfc0, 0xdfe0,
3840                 0xe000, 0x11088,
3841                 0x1109c, 0x11110,
3842                 0x11118, 0x1117c,
3843                 0x11190, 0x11204,
3844                 0x19040, 0x1906c,
3845                 0x19078, 0x19080,
3846                 0x1908c, 0x19124,
3847                 0x19150, 0x191b0,
3848                 0x191d0, 0x191e8,
3849                 0x19238, 0x19290,
3850                 0x193f8, 0x19474,
3851                 0x19490, 0x194cc,
3852                 0x194f0, 0x194f8,
3853                 0x19c00, 0x19c60,
3854                 0x19c94, 0x19e10,
3855                 0x19e50, 0x19f34,
3856                 0x19f40, 0x19f50,
3857                 0x19f90, 0x19fe4,
3858                 0x1a000, 0x1a06c,
3859                 0x1a0b0, 0x1a120,
3860                 0x1a128, 0x1a138,
3861                 0x1a190, 0x1a1c4,
3862                 0x1a1fc, 0x1a1fc,
3863                 0x1e008, 0x1e00c,
3864                 0x1e040, 0x1e04c,
3865                 0x1e284, 0x1e290,
3866                 0x1e2c0, 0x1e2c0,
3867                 0x1e2e0, 0x1e2e0,
3868                 0x1e300, 0x1e384,
3869                 0x1e3c0, 0x1e3c8,
3870                 0x1e408, 0x1e40c,
3871                 0x1e440, 0x1e44c,
3872                 0x1e684, 0x1e690,
3873                 0x1e6c0, 0x1e6c0,
3874                 0x1e6e0, 0x1e6e0,
3875                 0x1e700, 0x1e784,
3876                 0x1e7c0, 0x1e7c8,
3877                 0x1e808, 0x1e80c,
3878                 0x1e840, 0x1e84c,
3879                 0x1ea84, 0x1ea90,
3880                 0x1eac0, 0x1eac0,
3881                 0x1eae0, 0x1eae0,
3882                 0x1eb00, 0x1eb84,
3883                 0x1ebc0, 0x1ebc8,
3884                 0x1ec08, 0x1ec0c,
3885                 0x1ec40, 0x1ec4c,
3886                 0x1ee84, 0x1ee90,
3887                 0x1eec0, 0x1eec0,
3888                 0x1eee0, 0x1eee0,
3889                 0x1ef00, 0x1ef84,
3890                 0x1efc0, 0x1efc8,
3891                 0x1f008, 0x1f00c,
3892                 0x1f040, 0x1f04c,
3893                 0x1f284, 0x1f290,
3894                 0x1f2c0, 0x1f2c0,
3895                 0x1f2e0, 0x1f2e0,
3896                 0x1f300, 0x1f384,
3897                 0x1f3c0, 0x1f3c8,
3898                 0x1f408, 0x1f40c,
3899                 0x1f440, 0x1f44c,
3900                 0x1f684, 0x1f690,
3901                 0x1f6c0, 0x1f6c0,
3902                 0x1f6e0, 0x1f6e0,
3903                 0x1f700, 0x1f784,
3904                 0x1f7c0, 0x1f7c8,
3905                 0x1f808, 0x1f80c,
3906                 0x1f840, 0x1f84c,
3907                 0x1fa84, 0x1fa90,
3908                 0x1fac0, 0x1fac0,
3909                 0x1fae0, 0x1fae0,
3910                 0x1fb00, 0x1fb84,
3911                 0x1fbc0, 0x1fbc8,
3912                 0x1fc08, 0x1fc0c,
3913                 0x1fc40, 0x1fc4c,
3914                 0x1fe84, 0x1fe90,
3915                 0x1fec0, 0x1fec0,
3916                 0x1fee0, 0x1fee0,
3917                 0x1ff00, 0x1ff84,
3918                 0x1ffc0, 0x1ffc8,
3919                 0x30000, 0x30030,
3920                 0x30100, 0x30144,
3921                 0x30190, 0x301d0,
3922                 0x30200, 0x30318,
3923                 0x30400, 0x3052c,
3924                 0x30540, 0x3061c,
3925                 0x30800, 0x30834,
3926                 0x308c0, 0x30908,
3927                 0x30910, 0x309ac,
3928                 0x30a00, 0x30a2c,
3929                 0x30a44, 0x30a50,
3930                 0x30a74, 0x30c24,
3931                 0x30d00, 0x30d00,
3932                 0x30d08, 0x30d14,
3933                 0x30d1c, 0x30d20,
3934                 0x30d3c, 0x30d50,
3935                 0x31200, 0x3120c,
3936                 0x31220, 0x31220,
3937                 0x31240, 0x31240,
3938                 0x31600, 0x3160c,
3939                 0x31a00, 0x31a1c,
3940                 0x31e00, 0x31e20,
3941                 0x31e38, 0x31e3c,
3942                 0x31e80, 0x31e80,
3943                 0x31e88, 0x31ea8,
3944                 0x31eb0, 0x31eb4,
3945                 0x31ec8, 0x31ed4,
3946                 0x31fb8, 0x32004,
3947                 0x32200, 0x32200,
3948                 0x32208, 0x32240,
3949                 0x32248, 0x32280,
3950                 0x32288, 0x322c0,
3951                 0x322c8, 0x322fc,
3952                 0x32600, 0x32630,
3953                 0x32a00, 0x32abc,
3954                 0x32b00, 0x32b70,
3955                 0x33000, 0x33048,
3956                 0x33060, 0x3309c,
3957                 0x330f0, 0x33148,
3958                 0x33160, 0x3319c,
3959                 0x331f0, 0x332e4,
3960                 0x332f8, 0x333e4,
3961                 0x333f8, 0x33448,
3962                 0x33460, 0x3349c,
3963                 0x334f0, 0x33548,
3964                 0x33560, 0x3359c,
3965                 0x335f0, 0x336e4,
3966                 0x336f8, 0x337e4,
3967                 0x337f8, 0x337fc,
3968                 0x33814, 0x33814,
3969                 0x3382c, 0x3382c,
3970                 0x33880, 0x3388c,
3971                 0x338e8, 0x338ec,
3972                 0x33900, 0x33948,
3973                 0x33960, 0x3399c,
3974                 0x339f0, 0x33ae4,
3975                 0x33af8, 0x33b10,
3976                 0x33b28, 0x33b28,
3977                 0x33b3c, 0x33b50,
3978                 0x33bf0, 0x33c10,
3979                 0x33c28, 0x33c28,
3980                 0x33c3c, 0x33c50,
3981                 0x33cf0, 0x33cfc,
3982                 0x34000, 0x34030,
3983                 0x34100, 0x34144,
3984                 0x34190, 0x341d0,
3985                 0x34200, 0x34318,
3986                 0x34400, 0x3452c,
3987                 0x34540, 0x3461c,
3988                 0x34800, 0x34834,
3989                 0x348c0, 0x34908,
3990                 0x34910, 0x349ac,
3991                 0x34a00, 0x34a2c,
3992                 0x34a44, 0x34a50,
3993                 0x34a74, 0x34c24,
3994                 0x34d00, 0x34d00,
3995                 0x34d08, 0x34d14,
3996                 0x34d1c, 0x34d20,
3997                 0x34d3c, 0x34d50,
3998                 0x35200, 0x3520c,
3999                 0x35220, 0x35220,
4000                 0x35240, 0x35240,
4001                 0x35600, 0x3560c,
4002                 0x35a00, 0x35a1c,
4003                 0x35e00, 0x35e20,
4004                 0x35e38, 0x35e3c,
4005                 0x35e80, 0x35e80,
4006                 0x35e88, 0x35ea8,
4007                 0x35eb0, 0x35eb4,
4008                 0x35ec8, 0x35ed4,
4009                 0x35fb8, 0x36004,
4010                 0x36200, 0x36200,
4011                 0x36208, 0x36240,
4012                 0x36248, 0x36280,
4013                 0x36288, 0x362c0,
4014                 0x362c8, 0x362fc,
4015                 0x36600, 0x36630,
4016                 0x36a00, 0x36abc,
4017                 0x36b00, 0x36b70,
4018                 0x37000, 0x37048,
4019                 0x37060, 0x3709c,
4020                 0x370f0, 0x37148,
4021                 0x37160, 0x3719c,
4022                 0x371f0, 0x372e4,
4023                 0x372f8, 0x373e4,
4024                 0x373f8, 0x37448,
4025                 0x37460, 0x3749c,
4026                 0x374f0, 0x37548,
4027                 0x37560, 0x3759c,
4028                 0x375f0, 0x376e4,
4029                 0x376f8, 0x377e4,
4030                 0x377f8, 0x377fc,
4031                 0x37814, 0x37814,
4032                 0x3782c, 0x3782c,
4033                 0x37880, 0x3788c,
4034                 0x378e8, 0x378ec,
4035                 0x37900, 0x37948,
4036                 0x37960, 0x3799c,
4037                 0x379f0, 0x37ae4,
4038                 0x37af8, 0x37b10,
4039                 0x37b28, 0x37b28,
4040                 0x37b3c, 0x37b50,
4041                 0x37bf0, 0x37c10,
4042                 0x37c28, 0x37c28,
4043                 0x37c3c, 0x37c50,
4044                 0x37cf0, 0x37cfc,
4045                 0x38000, 0x38030,
4046                 0x38100, 0x38144,
4047                 0x38190, 0x381d0,
4048                 0x38200, 0x38318,
4049                 0x38400, 0x3852c,
4050                 0x38540, 0x3861c,
4051                 0x38800, 0x38834,
4052                 0x388c0, 0x38908,
4053                 0x38910, 0x389ac,
4054                 0x38a00, 0x38a2c,
4055                 0x38a44, 0x38a50,
4056                 0x38a74, 0x38c24,
4057                 0x38d00, 0x38d00,
4058                 0x38d08, 0x38d14,
4059                 0x38d1c, 0x38d20,
4060                 0x38d3c, 0x38d50,
4061                 0x39200, 0x3920c,
4062                 0x39220, 0x39220,
4063                 0x39240, 0x39240,
4064                 0x39600, 0x3960c,
4065                 0x39a00, 0x39a1c,
4066                 0x39e00, 0x39e20,
4067                 0x39e38, 0x39e3c,
4068                 0x39e80, 0x39e80,
4069                 0x39e88, 0x39ea8,
4070                 0x39eb0, 0x39eb4,
4071                 0x39ec8, 0x39ed4,
4072                 0x39fb8, 0x3a004,
4073                 0x3a200, 0x3a200,
4074                 0x3a208, 0x3a240,
4075                 0x3a248, 0x3a280,
4076                 0x3a288, 0x3a2c0,
4077                 0x3a2c8, 0x3a2fc,
4078                 0x3a600, 0x3a630,
4079                 0x3aa00, 0x3aabc,
4080                 0x3ab00, 0x3ab70,
4081                 0x3b000, 0x3b048,
4082                 0x3b060, 0x3b09c,
4083                 0x3b0f0, 0x3b148,
4084                 0x3b160, 0x3b19c,
4085                 0x3b1f0, 0x3b2e4,
4086                 0x3b2f8, 0x3b3e4,
4087                 0x3b3f8, 0x3b448,
4088                 0x3b460, 0x3b49c,
4089                 0x3b4f0, 0x3b548,
4090                 0x3b560, 0x3b59c,
4091                 0x3b5f0, 0x3b6e4,
4092                 0x3b6f8, 0x3b7e4,
4093                 0x3b7f8, 0x3b7fc,
4094                 0x3b814, 0x3b814,
4095                 0x3b82c, 0x3b82c,
4096                 0x3b880, 0x3b88c,
4097                 0x3b8e8, 0x3b8ec,
4098                 0x3b900, 0x3b948,
4099                 0x3b960, 0x3b99c,
4100                 0x3b9f0, 0x3bae4,
4101                 0x3baf8, 0x3bb10,
4102                 0x3bb28, 0x3bb28,
4103                 0x3bb3c, 0x3bb50,
4104                 0x3bbf0, 0x3bc10,
4105                 0x3bc28, 0x3bc28,
4106                 0x3bc3c, 0x3bc50,
4107                 0x3bcf0, 0x3bcfc,
4108                 0x3c000, 0x3c030,
4109                 0x3c100, 0x3c144,
4110                 0x3c190, 0x3c1d0,
4111                 0x3c200, 0x3c318,
4112                 0x3c400, 0x3c52c,
4113                 0x3c540, 0x3c61c,
4114                 0x3c800, 0x3c834,
4115                 0x3c8c0, 0x3c908,
4116                 0x3c910, 0x3c9ac,
4117                 0x3ca00, 0x3ca2c,
4118                 0x3ca44, 0x3ca50,
4119                 0x3ca74, 0x3cc24,
4120                 0x3cd00, 0x3cd00,
4121                 0x3cd08, 0x3cd14,
4122                 0x3cd1c, 0x3cd20,
4123                 0x3cd3c, 0x3cd50,
4124                 0x3d200, 0x3d20c,
4125                 0x3d220, 0x3d220,
4126                 0x3d240, 0x3d240,
4127                 0x3d600, 0x3d60c,
4128                 0x3da00, 0x3da1c,
4129                 0x3de00, 0x3de20,
4130                 0x3de38, 0x3de3c,
4131                 0x3de80, 0x3de80,
4132                 0x3de88, 0x3dea8,
4133                 0x3deb0, 0x3deb4,
4134                 0x3dec8, 0x3ded4,
4135                 0x3dfb8, 0x3e004,
4136                 0x3e200, 0x3e200,
4137                 0x3e208, 0x3e240,
4138                 0x3e248, 0x3e280,
4139                 0x3e288, 0x3e2c0,
4140                 0x3e2c8, 0x3e2fc,
4141                 0x3e600, 0x3e630,
4142                 0x3ea00, 0x3eabc,
4143                 0x3eb00, 0x3eb70,
4144                 0x3f000, 0x3f048,
4145                 0x3f060, 0x3f09c,
4146                 0x3f0f0, 0x3f148,
4147                 0x3f160, 0x3f19c,
4148                 0x3f1f0, 0x3f2e4,
4149                 0x3f2f8, 0x3f3e4,
4150                 0x3f3f8, 0x3f448,
4151                 0x3f460, 0x3f49c,
4152                 0x3f4f0, 0x3f548,
4153                 0x3f560, 0x3f59c,
4154                 0x3f5f0, 0x3f6e4,
4155                 0x3f6f8, 0x3f7e4,
4156                 0x3f7f8, 0x3f7fc,
4157                 0x3f814, 0x3f814,
4158                 0x3f82c, 0x3f82c,
4159                 0x3f880, 0x3f88c,
4160                 0x3f8e8, 0x3f8ec,
4161                 0x3f900, 0x3f948,
4162                 0x3f960, 0x3f99c,
4163                 0x3f9f0, 0x3fae4,
4164                 0x3faf8, 0x3fb10,
4165                 0x3fb28, 0x3fb28,
4166                 0x3fb3c, 0x3fb50,
4167                 0x3fbf0, 0x3fc10,
4168                 0x3fc28, 0x3fc28,
4169                 0x3fc3c, 0x3fc50,
4170                 0x3fcf0, 0x3fcfc,
4171                 0x40000, 0x4000c,
4172                 0x40040, 0x40068,
4173                 0x4007c, 0x40144,
4174                 0x40180, 0x4018c,
4175                 0x40200, 0x40298,
4176                 0x402ac, 0x4033c,
4177                 0x403f8, 0x403fc,
4178                 0x41304, 0x413c4,
4179                 0x41400, 0x4141c,
4180                 0x41480, 0x414d0,
4181                 0x44000, 0x44078,
4182                 0x440c0, 0x44278,
4183                 0x442c0, 0x44478,
4184                 0x444c0, 0x44678,
4185                 0x446c0, 0x44878,
4186                 0x448c0, 0x449fc,
4187                 0x45000, 0x45068,
4188                 0x45080, 0x45084,
4189                 0x450a0, 0x450b0,
4190                 0x45200, 0x45268,
4191                 0x45280, 0x45284,
4192                 0x452a0, 0x452b0,
4193                 0x460c0, 0x460e4,
4194                 0x47000, 0x4708c,
4195                 0x47200, 0x47250,
4196                 0x47400, 0x47420,
4197                 0x47600, 0x47618,
4198                 0x47800, 0x47814,
4199                 0x48000, 0x4800c,
4200                 0x48040, 0x48068,
4201                 0x4807c, 0x48144,
4202                 0x48180, 0x4818c,
4203                 0x48200, 0x48298,
4204                 0x482ac, 0x4833c,
4205                 0x483f8, 0x483fc,
4206                 0x49304, 0x493c4,
4207                 0x49400, 0x4941c,
4208                 0x49480, 0x494d0,
4209                 0x4c000, 0x4c078,
4210                 0x4c0c0, 0x4c278,
4211                 0x4c2c0, 0x4c478,
4212                 0x4c4c0, 0x4c678,
4213                 0x4c6c0, 0x4c878,
4214                 0x4c8c0, 0x4c9fc,
4215                 0x4d000, 0x4d068,
4216                 0x4d080, 0x4d084,
4217                 0x4d0a0, 0x4d0b0,
4218                 0x4d200, 0x4d268,
4219                 0x4d280, 0x4d284,
4220                 0x4d2a0, 0x4d2b0,
4221                 0x4e0c0, 0x4e0e4,
4222                 0x4f000, 0x4f08c,
4223                 0x4f200, 0x4f250,
4224                 0x4f400, 0x4f420,
4225                 0x4f600, 0x4f618,
4226                 0x4f800, 0x4f814,
4227                 0x50000, 0x500cc,
4228                 0x50400, 0x50400,
4229                 0x50800, 0x508cc,
4230                 0x50c00, 0x50c00,
4231                 0x51000, 0x5101c,
4232                 0x51300, 0x51308,
4233         };
4234
4235         if (is_t4(sc)) {
4236                 reg_ranges = &t4_reg_ranges[0];
4237                 n = nitems(t4_reg_ranges);
4238         } else {
4239                 reg_ranges = &t5_reg_ranges[0];
4240                 n = nitems(t5_reg_ranges);
4241         }
4242
4243         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4244         for (i = 0; i < n; i += 2)
4245                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4246 }
4247
4248 static void
4249 cxgbe_tick(void *arg)
4250 {
4251         struct port_info *pi = arg;
4252         struct adapter *sc = pi->adapter;
4253         struct ifnet *ifp = pi->ifp;
4254         struct sge_txq *txq;
4255         int i, drops;
4256         struct port_stats *s = &pi->stats;
4257
4258         PORT_LOCK(pi);
4259         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4260                 PORT_UNLOCK(pi);
4261                 return; /* without scheduling another callout */
4262         }
4263
4264         t4_get_port_stats(sc, pi->tx_chan, s);
4265
4266         ifp->if_opackets = s->tx_frames - s->tx_pause;
4267         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4268         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4269         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4270         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4271         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4272         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4273             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4274             s->rx_trunc3;
4275         for (i = 0; i < 4; i++) {
4276                 if (pi->rx_chan_map & (1 << i)) {
4277                         uint32_t v;
4278
4279                         /*
4280                          * XXX: indirect reads from the same ADDR/DATA pair can
4281                          * race with each other.
4282                          */
4283                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4284                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4285                         ifp->if_iqdrops += v;
4286                 }
4287         }
4288
4289         drops = s->tx_drop;
4290         for_each_txq(pi, i, txq)
4291                 drops += txq->br->br_drops;
4292         ifp->if_oqdrops = drops;
4293
4294         ifp->if_oerrors = s->tx_error_frames;
4295         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4296             s->rx_fcs_err + s->rx_len_err;
4297
4298         callout_schedule(&pi->tick, hz);
4299         PORT_UNLOCK(pi);
4300 }
4301
4302 static void
4303 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4304 {
4305         struct ifnet *vlan;
4306
4307         if (arg != ifp || ifp->if_type != IFT_ETHER)
4308                 return;
4309
4310         vlan = VLAN_DEVAT(ifp, vid);
4311         VLAN_SETCOOKIE(vlan, ifp);
4312 }
4313
4314 static int
4315 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4316 {
4317
4318 #ifdef INVARIANTS
4319         panic("%s: opcode 0x%02x on iq %p with payload %p",
4320             __func__, rss->opcode, iq, m);
4321 #else
4322         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4323             __func__, rss->opcode, iq, m);
4324         m_freem(m);
4325 #endif
4326         return (EDOOFUS);
4327 }
4328
4329 int
4330 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4331 {
4332         uintptr_t *loc, new;
4333
4334         if (opcode >= nitems(sc->cpl_handler))
4335                 return (EINVAL);
4336
4337         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4338         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4339         atomic_store_rel_ptr(loc, new);
4340
4341         return (0);
4342 }
4343
4344 static int
4345 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4346 {
4347
4348 #ifdef INVARIANTS
4349         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4350 #else
4351         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4352             __func__, iq, ctrl);
4353 #endif
4354         return (EDOOFUS);
4355 }
4356
4357 int
4358 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4359 {
4360         uintptr_t *loc, new;
4361
4362         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4363         loc = (uintptr_t *) &sc->an_handler;
4364         atomic_store_rel_ptr(loc, new);
4365
4366         return (0);
4367 }
4368
4369 static int
4370 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4371 {
4372         const struct cpl_fw6_msg *cpl =
4373             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4374
4375 #ifdef INVARIANTS
4376         panic("%s: fw_msg type %d", __func__, cpl->type);
4377 #else
4378         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4379 #endif
4380         return (EDOOFUS);
4381 }
4382
4383 int
4384 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4385 {
4386         uintptr_t *loc, new;
4387
4388         if (type >= nitems(sc->fw_msg_handler))
4389                 return (EINVAL);
4390
4391         /*
4392          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4393          * handler dispatch table.  Reject any attempt to install a handler for
4394          * this subtype.
4395          */
4396         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4397                 return (EINVAL);
4398
4399         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4400         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4401         atomic_store_rel_ptr(loc, new);
4402
4403         return (0);
4404 }
4405
4406 static int
4407 t4_sysctls(struct adapter *sc)
4408 {
4409         struct sysctl_ctx_list *ctx;
4410         struct sysctl_oid *oid;
4411         struct sysctl_oid_list *children, *c0;
4412         static char *caps[] = {
4413                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4414                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4415                     "\6HASHFILTER\7ETHOFLD",
4416                 "\20\1TOE",                             /* caps[2] toecaps */
4417                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4418                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4419                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4420                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4421                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4422                     "\4PO_INITIAOR\5PO_TARGET"
4423         };
4424         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4425
4426         ctx = device_get_sysctl_ctx(sc->dev);
4427
4428         /*
4429          * dev.t4nex.X.
4430          */
4431         oid = device_get_sysctl_tree(sc->dev);
4432         c0 = children = SYSCTL_CHILDREN(oid);
4433
4434         sc->sc_do_rxcopy = 1;
4435         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4436             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4437
4438         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4439             sc->params.nports, "# of ports");
4440
4441         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4442             NULL, chip_rev(sc), "chip hardware revision");
4443
4444         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4445             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4446
4447         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4448             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4449
4450         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4451             sc->cfcsum, "config file checksum");
4452
4453         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4454             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4455             sysctl_bitfield, "A", "available doorbells");
4456
4457         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4458             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4459             sysctl_bitfield, "A", "available link capabilities");
4460
4461         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4462             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4463             sysctl_bitfield, "A", "available NIC capabilities");
4464
4465         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4466             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4467             sysctl_bitfield, "A", "available TCP offload capabilities");
4468
4469         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4470             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4471             sysctl_bitfield, "A", "available RDMA capabilities");
4472
4473         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4474             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4475             sysctl_bitfield, "A", "available iSCSI capabilities");
4476
4477         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4478             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4479             sysctl_bitfield, "A", "available FCoE capabilities");
4480
4481         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4482             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4483
4484         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4485             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4486             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4487             "interrupt holdoff timer values (us)");
4488
4489         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4490             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4491             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4492             "interrupt holdoff packet counter values");
4493
4494         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4495             NULL, sc->tids.nftids, "number of filters");
4496
4497         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4498             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4499             "chip temperature (in Celsius)");
4500
4501         t4_sge_sysctls(sc, ctx, children);
4502
4503         sc->lro_timeout = 100;
4504         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4505             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4506
4507 #ifdef SBUF_DRAIN
4508         /*
4509          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4510          */
4511         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4512             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4513             "logs and miscellaneous information");
4514         children = SYSCTL_CHILDREN(oid);
4515
4516         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4517             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4518             sysctl_cctrl, "A", "congestion control");
4519
4520         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4521             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4522             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4523
4524         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4525             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4526             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4527
4528         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4529             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4530             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4531
4532         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4533             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4534             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4535
4536         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4537             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4538             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4539
4540         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4541             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4542             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4543
4544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4545             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4546             sysctl_cim_la, "A", "CIM logic analyzer");
4547
4548         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4549             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4550             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4551
4552         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4553             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4554             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4555
4556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4557             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4558             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4559
4560         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4561             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4562             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4563
4564         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4565             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4566             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4567
4568         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4569             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4570             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4571
4572         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4573             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4574             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4575
4576         if (is_t5(sc)) {
4577                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4578                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4579                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4580
4581                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4582                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4583                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4584         }
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4592             sysctl_cim_qcfg, "A", "CIM queue configuration");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4596             sysctl_cpl_stats, "A", "CPL statistics");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4600             sysctl_ddp_stats, "A", "DDP statistics");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4604             sysctl_devlog, "A", "firmware's device log");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4608             sysctl_fcoe_stats, "A", "FCoE statistics");
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_hw_sched, "A", "hardware scheduler ");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_l2t, "A", "hardware L2 table");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_lb_stats, "A", "loopback statistics");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_meminfo, "A", "memory regions");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4628             sysctl_mps_tcam, "A", "MPS TCAM entries");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4632             sysctl_path_mtus, "A", "path MTUs");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4636             sysctl_pm_stats, "A", "PM statistics");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4640             sysctl_rdma_stats, "A", "RDMA statistics");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4644             sysctl_tcp_stats, "A", "TCP statistics");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4648             sysctl_tids, "A", "TID information");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4652             sysctl_tp_err_stats, "A", "TP error statistics");
4653
4654         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4655             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4656             sysctl_tp_la, "A", "TP logic analyzer");
4657
4658         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4659             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4660             sysctl_tx_rate, "A", "Tx rate");
4661
4662         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4663             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4664             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4665
4666         if (is_t5(sc)) {
4667                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4668                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4669                     sysctl_wcwr_stats, "A", "write combined work requests");
4670         }
4671 #endif
4672
4673 #ifdef TCP_OFFLOAD
4674         if (is_offload(sc)) {
4675                 /*
4676                  * dev.t4nex.X.toe.
4677                  */
4678                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4679                     NULL, "TOE parameters");
4680                 children = SYSCTL_CHILDREN(oid);
4681
4682                 sc->tt.sndbuf = 256 * 1024;
4683                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4684                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4685
4686                 sc->tt.ddp = 0;
4687                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4688                     &sc->tt.ddp, 0, "DDP allowed");
4689
4690                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4691                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4692                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4693
4694                 sc->tt.ddp_thres =
4695                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4696                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4697                     &sc->tt.ddp_thres, 0, "DDP threshold");
4698
4699                 sc->tt.rx_coalesce = 1;
4700                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4701                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4702         }
4703 #endif
4704
4705
4706         return (0);
4707 }
4708
4709 static int
4710 cxgbe_sysctls(struct port_info *pi)
4711 {
4712         struct sysctl_ctx_list *ctx;
4713         struct sysctl_oid *oid;
4714         struct sysctl_oid_list *children;
4715         struct adapter *sc = pi->adapter;
4716
4717         ctx = device_get_sysctl_ctx(pi->dev);
4718
4719         /*
4720          * dev.cxgbe.X.
4721          */
4722         oid = device_get_sysctl_tree(pi->dev);
4723         children = SYSCTL_CHILDREN(oid);
4724
4725         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4726            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4727         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4728                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4729                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4730                     "PHY temperature (in Celsius)");
4731                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4732                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4733                     "PHY firmware version");
4734         }
4735         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4736             &pi->nrxq, 0, "# of rx queues");
4737         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4738             &pi->ntxq, 0, "# of tx queues");
4739         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4740             &pi->first_rxq, 0, "index of first rx queue");
4741         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4742             &pi->first_txq, 0, "index of first tx queue");
4743         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4744             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4745             "Reserve queue 0 for non-flowid packets");
4746
4747 #ifdef TCP_OFFLOAD
4748         if (is_offload(sc)) {
4749                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4750                     &pi->nofldrxq, 0,
4751                     "# of rx queues for offloaded TCP connections");
4752                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4753                     &pi->nofldtxq, 0,
4754                     "# of tx queues for offloaded TCP connections");
4755                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4756                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4757                     "index of first TOE rx queue");
4758                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4759                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4760                     "index of first TOE tx queue");
4761         }
4762 #endif
4763 #ifdef DEV_NETMAP
4764         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4765             &pi->nnmrxq, 0, "# of rx queues for netmap");
4766         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4767             &pi->nnmtxq, 0, "# of tx queues for netmap");
4768         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4769             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4770             "index of first netmap rx queue");
4771         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4772             CTLFLAG_RD, &pi->first_nm_txq, 0,
4773             "index of first netmap tx queue");
4774 #endif
4775
4776         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4777             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4778             "holdoff timer index");
4779         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4780             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4781             "holdoff packet counter index");
4782
4783         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4784             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4785             "rx queue size");
4786         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4787             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4788             "tx queue size");
4789
4790         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4791             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4792             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4793
4794         /*
4795          * dev.cxgbe.X.stats.
4796          */
4797         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4798             NULL, "port statistics");
4799         children = SYSCTL_CHILDREN(oid);
4800
4801 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4802         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4803             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4804             sysctl_handle_t4_reg64, "QU", desc)
4805
4806         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4807             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4808         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4809             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4810         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4811             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4812         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4813             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4814         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4815             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4816         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4817             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4818         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4819             "# of tx frames in this range",
4820             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4821         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4822             "# of tx frames in this range",
4823             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4824         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4825             "# of tx frames in this range",
4826             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4827         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4828             "# of tx frames in this range",
4829             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4830         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4831             "# of tx frames in this range",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4834             "# of tx frames in this range",
4835             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4836         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4837             "# of tx frames in this range",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4843         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4844             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4845         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4846             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4847         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4848             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4849         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4850             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4851         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4853         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4855         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4857         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4858             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4859
4860         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4862         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4864         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4866         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4868         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4870         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4872         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4874         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4875             "# of frames received with bad FCS",
4876             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4877         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4878             "# of frames received with length error",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4880         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4882         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4884         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4885             "# of rx frames in this range",
4886             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4887         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4888             "# of rx frames in this range",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4890         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4891             "# of rx frames in this range",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4894             "# of rx frames in this range",
4895             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4896         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4897             "# of rx frames in this range",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4899         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4900             "# of rx frames in this range",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4902         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4903             "# of rx frames in this range",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4913         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4915         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4917         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4919         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4921         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4923
4924 #undef SYSCTL_ADD_T4_REG64
4925
4926 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4927         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4928             &pi->stats.name, desc)
4929
4930         /* We get these from port_stats and they may be stale by upto 1s */
4931         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4932             "# drops due to buffer-group 0 overflows");
4933         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4934             "# drops due to buffer-group 1 overflows");
4935         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4936             "# drops due to buffer-group 2 overflows");
4937         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4938             "# drops due to buffer-group 3 overflows");
4939         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4940             "# of buffer-group 0 truncated packets");
4941         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4942             "# of buffer-group 1 truncated packets");
4943         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4944             "# of buffer-group 2 truncated packets");
4945         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4946             "# of buffer-group 3 truncated packets");
4947
4948 #undef SYSCTL_ADD_T4_PORTSTAT
4949
4950         return (0);
4951 }
4952
4953 static int
4954 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4955 {
4956         int rc, *i;
4957         struct sbuf sb;
4958
4959         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4960         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4961                 sbuf_printf(&sb, "%d ", *i);
4962         sbuf_trim(&sb);
4963         sbuf_finish(&sb);
4964         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4965         sbuf_delete(&sb);
4966         return (rc);
4967 }
4968
4969 static int
4970 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4971 {
4972         int rc;
4973         struct sbuf *sb;
4974
4975         rc = sysctl_wire_old_buffer(req, 0);
4976         if (rc != 0)
4977                 return(rc);
4978
4979         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4980         if (sb == NULL)
4981                 return (ENOMEM);
4982
4983         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4984         rc = sbuf_finish(sb);
4985         sbuf_delete(sb);
4986
4987         return (rc);
4988 }
4989
4990 static int
4991 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4992 {
4993         struct port_info *pi = arg1;
4994         int op = arg2;
4995         struct adapter *sc = pi->adapter;
4996         u_int v;
4997         int rc;
4998
4999         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5000         if (rc)
5001                 return (rc);
5002         /* XXX: magic numbers */
5003         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5004             &v);
5005         end_synchronized_op(sc, 0);
5006         if (rc)
5007                 return (rc);
5008         if (op == 0)
5009                 v /= 256;
5010
5011         rc = sysctl_handle_int(oidp, &v, 0, req);
5012         return (rc);
5013 }
5014
5015 static int
5016 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5017 {
5018         struct port_info *pi = arg1;
5019         int rc, val;
5020
5021         val = pi->rsrv_noflowq;
5022         rc = sysctl_handle_int(oidp, &val, 0, req);
5023         if (rc != 0 || req->newptr == NULL)
5024                 return (rc);
5025
5026         if ((val >= 1) && (pi->ntxq > 1))
5027                 pi->rsrv_noflowq = 1;
5028         else
5029                 pi->rsrv_noflowq = 0;
5030
5031         return (rc);
5032 }
5033
5034 static int
5035 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5036 {
5037         struct port_info *pi = arg1;
5038         struct adapter *sc = pi->adapter;
5039         int idx, rc, i;
5040         struct sge_rxq *rxq;
5041 #ifdef TCP_OFFLOAD
5042         struct sge_ofld_rxq *ofld_rxq;
5043 #endif
5044         uint8_t v;
5045
5046         idx = pi->tmr_idx;
5047
5048         rc = sysctl_handle_int(oidp, &idx, 0, req);
5049         if (rc != 0 || req->newptr == NULL)
5050                 return (rc);
5051
5052         if (idx < 0 || idx >= SGE_NTIMERS)
5053                 return (EINVAL);
5054
5055         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5056             "t4tmr");
5057         if (rc)
5058                 return (rc);
5059
5060         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5061         for_each_rxq(pi, i, rxq) {
5062 #ifdef atomic_store_rel_8
5063                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5064 #else
5065                 rxq->iq.intr_params = v;
5066 #endif
5067         }
5068 #ifdef TCP_OFFLOAD
5069         for_each_ofld_rxq(pi, i, ofld_rxq) {
5070 #ifdef atomic_store_rel_8
5071                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5072 #else
5073                 ofld_rxq->iq.intr_params = v;
5074 #endif
5075         }
5076 #endif
5077         pi->tmr_idx = idx;
5078
5079         end_synchronized_op(sc, LOCK_HELD);
5080         return (0);
5081 }
5082
5083 static int
5084 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5085 {
5086         struct port_info *pi = arg1;
5087         struct adapter *sc = pi->adapter;
5088         int idx, rc;
5089
5090         idx = pi->pktc_idx;
5091
5092         rc = sysctl_handle_int(oidp, &idx, 0, req);
5093         if (rc != 0 || req->newptr == NULL)
5094                 return (rc);
5095
5096         if (idx < -1 || idx >= SGE_NCOUNTERS)
5097                 return (EINVAL);
5098
5099         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5100             "t4pktc");
5101         if (rc)
5102                 return (rc);
5103
5104         if (pi->flags & PORT_INIT_DONE)
5105                 rc = EBUSY; /* cannot be changed once the queues are created */
5106         else
5107                 pi->pktc_idx = idx;
5108
5109         end_synchronized_op(sc, LOCK_HELD);
5110         return (rc);
5111 }
5112
5113 static int
5114 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5115 {
5116         struct port_info *pi = arg1;
5117         struct adapter *sc = pi->adapter;
5118         int qsize, rc;
5119
5120         qsize = pi->qsize_rxq;
5121
5122         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5123         if (rc != 0 || req->newptr == NULL)
5124                 return (rc);
5125
5126         if (qsize < 128 || (qsize & 7))
5127                 return (EINVAL);
5128
5129         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5130             "t4rxqs");
5131         if (rc)
5132                 return (rc);
5133
5134         if (pi->flags & PORT_INIT_DONE)
5135                 rc = EBUSY; /* cannot be changed once the queues are created */
5136         else
5137                 pi->qsize_rxq = qsize;
5138
5139         end_synchronized_op(sc, LOCK_HELD);
5140         return (rc);
5141 }
5142
5143 static int
5144 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5145 {
5146         struct port_info *pi = arg1;
5147         struct adapter *sc = pi->adapter;
5148         int qsize, rc;
5149
5150         qsize = pi->qsize_txq;
5151
5152         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5153         if (rc != 0 || req->newptr == NULL)
5154                 return (rc);
5155
5156         /* bufring size must be powerof2 */
5157         if (qsize < 128 || !powerof2(qsize))
5158                 return (EINVAL);
5159
5160         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5161             "t4txqs");
5162         if (rc)
5163                 return (rc);
5164
5165         if (pi->flags & PORT_INIT_DONE)
5166                 rc = EBUSY; /* cannot be changed once the queues are created */
5167         else
5168                 pi->qsize_txq = qsize;
5169
5170         end_synchronized_op(sc, LOCK_HELD);
5171         return (rc);
5172 }
5173
5174 static int
5175 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5176 {
5177         struct port_info *pi = arg1;
5178         struct adapter *sc = pi->adapter;
5179         struct link_config *lc = &pi->link_cfg;
5180         int rc;
5181
5182         if (req->newptr == NULL) {
5183                 struct sbuf *sb;
5184                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5185
5186                 rc = sysctl_wire_old_buffer(req, 0);
5187                 if (rc != 0)
5188                         return(rc);
5189
5190                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5191                 if (sb == NULL)
5192                         return (ENOMEM);
5193
5194                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5195                 rc = sbuf_finish(sb);
5196                 sbuf_delete(sb);
5197         } else {
5198                 char s[2];
5199                 int n;
5200
5201                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5202                 s[1] = 0;
5203
5204                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5205                 if (rc != 0)
5206                         return(rc);
5207
5208                 if (s[1] != 0)
5209                         return (EINVAL);
5210                 if (s[0] < '0' || s[0] > '9')
5211                         return (EINVAL);        /* not a number */
5212                 n = s[0] - '0';
5213                 if (n & ~(PAUSE_TX | PAUSE_RX))
5214                         return (EINVAL);        /* some other bit is set too */
5215
5216                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5217                 if (rc)
5218                         return (rc);
5219                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5220                         int link_ok = lc->link_ok;
5221
5222                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5223                         lc->requested_fc |= n;
5224                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5225                         lc->link_ok = link_ok;  /* restore */
5226                 }
5227                 end_synchronized_op(sc, 0);
5228         }
5229
5230         return (rc);
5231 }
5232
5233 static int
5234 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5235 {
5236         struct adapter *sc = arg1;
5237         int reg = arg2;
5238         uint64_t val;
5239
5240         val = t4_read_reg64(sc, reg);
5241
5242         return (sysctl_handle_64(oidp, &val, 0, req));
5243 }
5244
5245 static int
5246 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5247 {
5248         struct adapter *sc = arg1;
5249         int rc, t;
5250         uint32_t param, val;
5251
5252         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5253         if (rc)
5254                 return (rc);
5255         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5256             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5257             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5258         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5259         end_synchronized_op(sc, 0);
5260         if (rc)
5261                 return (rc);
5262
5263         /* unknown is returned as 0 but we display -1 in that case */
5264         t = val == 0 ? -1 : val;
5265
5266         rc = sysctl_handle_int(oidp, &t, 0, req);
5267         return (rc);
5268 }
5269
5270 #ifdef SBUF_DRAIN
5271 static int
5272 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5273 {
5274         struct adapter *sc = arg1;
5275         struct sbuf *sb;
5276         int rc, i;
5277         uint16_t incr[NMTUS][NCCTRL_WIN];
5278         static const char *dec_fac[] = {
5279                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5280                 "0.9375"
5281         };
5282
5283         rc = sysctl_wire_old_buffer(req, 0);
5284         if (rc != 0)
5285                 return (rc);
5286
5287         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5288         if (sb == NULL)
5289                 return (ENOMEM);
5290
5291         t4_read_cong_tbl(sc, incr);
5292
5293         for (i = 0; i < NCCTRL_WIN; ++i) {
5294                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5295                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5296                     incr[5][i], incr[6][i], incr[7][i]);
5297                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5298                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5299                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5300                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5301         }
5302
5303         rc = sbuf_finish(sb);
5304         sbuf_delete(sb);
5305
5306         return (rc);
5307 }
5308
5309 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5310         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5311         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5312         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5313 };
5314
5315 static int
5316 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5317 {
5318         struct adapter *sc = arg1;
5319         struct sbuf *sb;
5320         int rc, i, n, qid = arg2;
5321         uint32_t *buf, *p;
5322         char *qtype;
5323         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5324
5325         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5326             ("%s: bad qid %d\n", __func__, qid));
5327
5328         if (qid < CIM_NUM_IBQ) {
5329                 /* inbound queue */
5330                 qtype = "IBQ";
5331                 n = 4 * CIM_IBQ_SIZE;
5332                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5333                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5334         } else {
5335                 /* outbound queue */
5336                 qtype = "OBQ";
5337                 qid -= CIM_NUM_IBQ;
5338                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5339                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5340                 rc = t4_read_cim_obq(sc, qid, buf, n);
5341         }
5342
5343         if (rc < 0) {
5344                 rc = -rc;
5345                 goto done;
5346         }
5347         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5348
5349         rc = sysctl_wire_old_buffer(req, 0);
5350         if (rc != 0)
5351                 goto done;
5352
5353         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5354         if (sb == NULL) {
5355                 rc = ENOMEM;
5356                 goto done;
5357         }
5358
5359         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5360         for (i = 0, p = buf; i < n; i += 16, p += 4)
5361                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5362                     p[2], p[3]);
5363
5364         rc = sbuf_finish(sb);
5365         sbuf_delete(sb);
5366 done:
5367         free(buf, M_CXGBE);
5368         return (rc);
5369 }
5370
5371 static int
5372 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5373 {
5374         struct adapter *sc = arg1;
5375         u_int cfg;
5376         struct sbuf *sb;
5377         uint32_t *buf, *p;
5378         int rc;
5379
5380         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5381         if (rc != 0)
5382                 return (rc);
5383
5384         rc = sysctl_wire_old_buffer(req, 0);
5385         if (rc != 0)
5386                 return (rc);
5387
5388         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5389         if (sb == NULL)
5390                 return (ENOMEM);
5391
5392         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5393             M_ZERO | M_WAITOK);
5394
5395         rc = -t4_cim_read_la(sc, buf, NULL);
5396         if (rc != 0)
5397                 goto done;
5398
5399         sbuf_printf(sb, "Status   Data      PC%s",
5400             cfg & F_UPDBGLACAPTPCONLY ? "" :
5401             "     LS0Stat  LS0Addr             LS0Data");
5402
5403         KASSERT((sc->params.cim_la_size & 7) == 0,
5404             ("%s: p will walk off the end of buf", __func__));
5405
5406         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5407                 if (cfg & F_UPDBGLACAPTPCONLY) {
5408                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5409                             p[6], p[7]);
5410                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5411                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5412                             p[4] & 0xff, p[5] >> 8);
5413                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5414                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5415                             p[1] & 0xf, p[2] >> 4);
5416                 } else {
5417                         sbuf_printf(sb,
5418                             "\n  %02x   %x%07x %x%07x %08x %08x "
5419                             "%08x%08x%08x%08x",
5420                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5421                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5422                             p[6], p[7]);
5423                 }
5424         }
5425
5426         rc = sbuf_finish(sb);
5427         sbuf_delete(sb);
5428 done:
5429         free(buf, M_CXGBE);
5430         return (rc);
5431 }
5432
5433 static int
5434 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5435 {
5436         struct adapter *sc = arg1;
5437         u_int i;
5438         struct sbuf *sb;
5439         uint32_t *buf, *p;
5440         int rc;
5441
5442         rc = sysctl_wire_old_buffer(req, 0);
5443         if (rc != 0)
5444                 return (rc);
5445
5446         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5447         if (sb == NULL)
5448                 return (ENOMEM);
5449
5450         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5451             M_ZERO | M_WAITOK);
5452
5453         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5454         p = buf;
5455
5456         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5457                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5458                     p[1], p[0]);
5459         }
5460
5461         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5462         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5463                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5464                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5465                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5466                     (p[1] >> 2) | ((p[2] & 3) << 30),
5467                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5468                     p[0] & 1);
5469         }
5470
5471         rc = sbuf_finish(sb);
5472         sbuf_delete(sb);
5473         free(buf, M_CXGBE);
5474         return (rc);
5475 }
5476
5477 static int
5478 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5479 {
5480         struct adapter *sc = arg1;
5481         u_int i;
5482         struct sbuf *sb;
5483         uint32_t *buf, *p;
5484         int rc;
5485
5486         rc = sysctl_wire_old_buffer(req, 0);
5487         if (rc != 0)
5488                 return (rc);
5489
5490         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5491         if (sb == NULL)
5492                 return (ENOMEM);
5493
5494         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5495             M_ZERO | M_WAITOK);
5496
5497         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5498         p = buf;
5499
5500         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5501         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5502                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5503                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5504                     p[4], p[3], p[2], p[1], p[0]);
5505         }
5506
5507         sbuf_printf(sb, "\n\nCntl ID               Data");
5508         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5509                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5510                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5511         }
5512
5513         rc = sbuf_finish(sb);
5514         sbuf_delete(sb);
5515         free(buf, M_CXGBE);
5516         return (rc);
5517 }
5518
5519 static int
5520 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5521 {
5522         struct adapter *sc = arg1;
5523         struct sbuf *sb;
5524         int rc, i;
5525         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5526         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5527         uint16_t thres[CIM_NUM_IBQ];
5528         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5529         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5530         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5531
5532         if (is_t4(sc)) {
5533                 cim_num_obq = CIM_NUM_OBQ;
5534                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5535                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5536         } else {
5537                 cim_num_obq = CIM_NUM_OBQ_T5;
5538                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5539                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5540         }
5541         nq = CIM_NUM_IBQ + cim_num_obq;
5542
5543         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5544         if (rc == 0)
5545                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5546         if (rc != 0)
5547                 return (rc);
5548
5549         t4_read_cimq_cfg(sc, base, size, thres);
5550
5551         rc = sysctl_wire_old_buffer(req, 0);
5552         if (rc != 0)
5553                 return (rc);
5554
5555         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5556         if (sb == NULL)
5557                 return (ENOMEM);
5558
5559         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5560
5561         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5562                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5563                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5564                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5565                     G_QUEREMFLITS(p[2]) * 16);
5566         for ( ; i < nq; i++, p += 4, wr += 2)
5567                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5568                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5569                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5570                     G_QUEREMFLITS(p[2]) * 16);
5571
5572         rc = sbuf_finish(sb);
5573         sbuf_delete(sb);
5574
5575         return (rc);
5576 }
5577
5578 static int
5579 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5580 {
5581         struct adapter *sc = arg1;
5582         struct sbuf *sb;
5583         int rc;
5584         struct tp_cpl_stats stats;
5585
5586         rc = sysctl_wire_old_buffer(req, 0);
5587         if (rc != 0)
5588                 return (rc);
5589
5590         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5591         if (sb == NULL)
5592                 return (ENOMEM);
5593
5594         t4_tp_get_cpl_stats(sc, &stats);
5595
5596         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5597             "channel 3\n");
5598         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5599                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5600         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5601                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5602
5603         rc = sbuf_finish(sb);
5604         sbuf_delete(sb);
5605
5606         return (rc);
5607 }
5608
5609 static int
5610 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5611 {
5612         struct adapter *sc = arg1;
5613         struct sbuf *sb;
5614         int rc;
5615         struct tp_usm_stats stats;
5616
5617         rc = sysctl_wire_old_buffer(req, 0);
5618         if (rc != 0)
5619                 return(rc);
5620
5621         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5622         if (sb == NULL)
5623                 return (ENOMEM);
5624
5625         t4_get_usm_stats(sc, &stats);
5626
5627         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5628         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5629         sbuf_printf(sb, "Drops:  %u", stats.drops);
5630
5631         rc = sbuf_finish(sb);
5632         sbuf_delete(sb);
5633
5634         return (rc);
5635 }
5636
5637 const char *devlog_level_strings[] = {
5638         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5639         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5640         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5641         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5642         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5643         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5644 };
5645
5646 const char *devlog_facility_strings[] = {
5647         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5648         [FW_DEVLOG_FACILITY_CF]         = "CF",
5649         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5650         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5651         [FW_DEVLOG_FACILITY_RES]        = "RES",
5652         [FW_DEVLOG_FACILITY_HW]         = "HW",
5653         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5654         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5655         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5656         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5657         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5658         [FW_DEVLOG_FACILITY_VI]         = "VI",
5659         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5660         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5661         [FW_DEVLOG_FACILITY_TM]         = "TM",
5662         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5663         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5664         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5665         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5666         [FW_DEVLOG_FACILITY_RI]         = "RI",
5667         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5668         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5669         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5670         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5671 };
5672
5673 static int
5674 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5675 {
5676         struct adapter *sc = arg1;
5677         struct devlog_params *dparams = &sc->params.devlog;
5678         struct fw_devlog_e *buf, *e;
5679         int i, j, rc, nentries, first = 0, m;
5680         struct sbuf *sb;
5681         uint64_t ftstamp = UINT64_MAX;
5682
5683         if (dparams->start == 0) {
5684                 dparams->memtype = FW_MEMTYPE_EDC0;
5685                 dparams->start = 0x84000;
5686                 dparams->size = 32768;
5687         }
5688
5689         nentries = dparams->size / sizeof(struct fw_devlog_e);
5690
5691         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5692         if (buf == NULL)
5693                 return (ENOMEM);
5694
5695         m = fwmtype_to_hwmtype(dparams->memtype);
5696         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5697         if (rc != 0)
5698                 goto done;
5699
5700         for (i = 0; i < nentries; i++) {
5701                 e = &buf[i];
5702
5703                 if (e->timestamp == 0)
5704                         break;  /* end */
5705
5706                 e->timestamp = be64toh(e->timestamp);
5707                 e->seqno = be32toh(e->seqno);
5708                 for (j = 0; j < 8; j++)
5709                         e->params[j] = be32toh(e->params[j]);
5710
5711                 if (e->timestamp < ftstamp) {
5712                         ftstamp = e->timestamp;
5713                         first = i;
5714                 }
5715         }
5716
5717         if (buf[first].timestamp == 0)
5718                 goto done;      /* nothing in the log */
5719
5720         rc = sysctl_wire_old_buffer(req, 0);
5721         if (rc != 0)
5722                 goto done;
5723
5724         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5725         if (sb == NULL) {
5726                 rc = ENOMEM;
5727                 goto done;
5728         }
5729         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5730             "Seq#", "Tstamp", "Level", "Facility", "Message");
5731
5732         i = first;
5733         do {
5734                 e = &buf[i];
5735                 if (e->timestamp == 0)
5736                         break;  /* end */
5737
5738                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5739                     e->seqno, e->timestamp,
5740                     (e->level < nitems(devlog_level_strings) ?
5741                         devlog_level_strings[e->level] : "UNKNOWN"),
5742                     (e->facility < nitems(devlog_facility_strings) ?
5743                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5744                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5745                     e->params[2], e->params[3], e->params[4],
5746                     e->params[5], e->params[6], e->params[7]);
5747
5748                 if (++i == nentries)
5749                         i = 0;
5750         } while (i != first);
5751
5752         rc = sbuf_finish(sb);
5753         sbuf_delete(sb);
5754 done:
5755         free(buf, M_CXGBE);
5756         return (rc);
5757 }
5758
5759 static int
5760 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5761 {
5762         struct adapter *sc = arg1;
5763         struct sbuf *sb;
5764         int rc;
5765         struct tp_fcoe_stats stats[4];
5766
5767         rc = sysctl_wire_old_buffer(req, 0);
5768         if (rc != 0)
5769                 return (rc);
5770
5771         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5772         if (sb == NULL)
5773                 return (ENOMEM);
5774
5775         t4_get_fcoe_stats(sc, 0, &stats[0]);
5776         t4_get_fcoe_stats(sc, 1, &stats[1]);
5777         t4_get_fcoe_stats(sc, 2, &stats[2]);
5778         t4_get_fcoe_stats(sc, 3, &stats[3]);
5779
5780         sbuf_printf(sb, "                   channel 0        channel 1        "
5781             "channel 2        channel 3\n");
5782         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5783             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5784             stats[3].octetsDDP);
5785         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5786             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5787         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5788             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5789             stats[3].framesDrop);
5790
5791         rc = sbuf_finish(sb);
5792         sbuf_delete(sb);
5793
5794         return (rc);
5795 }
5796
5797 static int
5798 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5799 {
5800         struct adapter *sc = arg1;
5801         struct sbuf *sb;
5802         int rc, i;
5803         unsigned int map, kbps, ipg, mode;
5804         unsigned int pace_tab[NTX_SCHED];
5805
5806         rc = sysctl_wire_old_buffer(req, 0);
5807         if (rc != 0)
5808                 return (rc);
5809
5810         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5811         if (sb == NULL)
5812                 return (ENOMEM);
5813
5814         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5815         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5816         t4_read_pace_tbl(sc, pace_tab);
5817
5818         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5819             "Class IPG (0.1 ns)   Flow IPG (us)");
5820
5821         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5822                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5823                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5824                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5825                 if (kbps)
5826                         sbuf_printf(sb, "%9u     ", kbps);
5827                 else
5828                         sbuf_printf(sb, " disabled     ");
5829
5830                 if (ipg)
5831                         sbuf_printf(sb, "%13u        ", ipg);
5832                 else
5833                         sbuf_printf(sb, "     disabled        ");
5834
5835                 if (pace_tab[i])
5836                         sbuf_printf(sb, "%10u", pace_tab[i]);
5837                 else
5838                         sbuf_printf(sb, "  disabled");
5839         }
5840
5841         rc = sbuf_finish(sb);
5842         sbuf_delete(sb);
5843
5844         return (rc);
5845 }
5846
5847 static int
5848 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5849 {
5850         struct adapter *sc = arg1;
5851         struct sbuf *sb;
5852         int rc, i, j;
5853         uint64_t *p0, *p1;
5854         struct lb_port_stats s[2];
5855         static const char *stat_name[] = {
5856                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5857                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5858                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5859                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5860                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5861                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5862                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5863         };
5864
5865         rc = sysctl_wire_old_buffer(req, 0);
5866         if (rc != 0)
5867                 return (rc);
5868
5869         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5870         if (sb == NULL)
5871                 return (ENOMEM);
5872
5873         memset(s, 0, sizeof(s));
5874
5875         for (i = 0; i < 4; i += 2) {
5876                 t4_get_lb_stats(sc, i, &s[0]);
5877                 t4_get_lb_stats(sc, i + 1, &s[1]);
5878
5879                 p0 = &s[0].octets;
5880                 p1 = &s[1].octets;
5881                 sbuf_printf(sb, "%s                       Loopback %u"
5882                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5883
5884                 for (j = 0; j < nitems(stat_name); j++)
5885                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5886                                    *p0++, *p1++);
5887         }
5888
5889         rc = sbuf_finish(sb);
5890         sbuf_delete(sb);
5891
5892         return (rc);
5893 }
5894
5895 static int
5896 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5897 {
5898         int rc = 0;
5899         struct port_info *pi = arg1;
5900         struct sbuf *sb;
5901         static const char *linkdnreasons[] = {
5902                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5903                 "PHY overheated", "unknown", "rx los", "reserved7"
5904         };
5905
5906         rc = sysctl_wire_old_buffer(req, 0);
5907         if (rc != 0)
5908                 return(rc);
5909         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5910         if (sb == NULL)
5911                 return (ENOMEM);
5912
5913         if (pi->linkdnrc < 0)
5914                 sbuf_printf(sb, "n/a");
5915         else if (pi->linkdnrc < nitems(linkdnreasons))
5916                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5917         else
5918                 sbuf_printf(sb, "%d", pi->linkdnrc);
5919
5920         rc = sbuf_finish(sb);
5921         sbuf_delete(sb);
5922
5923         return (rc);
5924 }
5925
5926 struct mem_desc {
5927         unsigned int base;
5928         unsigned int limit;
5929         unsigned int idx;
5930 };
5931
5932 static int
5933 mem_desc_cmp(const void *a, const void *b)
5934 {
5935         return ((const struct mem_desc *)a)->base -
5936                ((const struct mem_desc *)b)->base;
5937 }
5938
5939 static void
5940 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5941     unsigned int to)
5942 {
5943         unsigned int size;
5944
5945         size = to - from + 1;
5946         if (size == 0)
5947                 return;
5948
5949         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5950         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5951 }
5952
5953 static int
5954 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5955 {
5956         struct adapter *sc = arg1;
5957         struct sbuf *sb;
5958         int rc, i, n;
5959         uint32_t lo, hi, used, alloc;
5960         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5961         static const char *region[] = {
5962                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5963                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5964                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5965                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5966                 "RQUDP region:", "PBL region:", "TXPBL region:",
5967                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5968                 "On-chip queues:"
5969         };
5970         struct mem_desc avail[4];
5971         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5972         struct mem_desc *md = mem;
5973
5974         rc = sysctl_wire_old_buffer(req, 0);
5975         if (rc != 0)
5976                 return (rc);
5977
5978         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5979         if (sb == NULL)
5980                 return (ENOMEM);
5981
5982         for (i = 0; i < nitems(mem); i++) {
5983                 mem[i].limit = 0;
5984                 mem[i].idx = i;
5985         }
5986
5987         /* Find and sort the populated memory ranges */
5988         i = 0;
5989         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5990         if (lo & F_EDRAM0_ENABLE) {
5991                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5992                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5993                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5994                 avail[i].idx = 0;
5995                 i++;
5996         }
5997         if (lo & F_EDRAM1_ENABLE) {
5998                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5999                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6000                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6001                 avail[i].idx = 1;
6002                 i++;
6003         }
6004         if (lo & F_EXT_MEM_ENABLE) {
6005                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6006                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6007                 avail[i].limit = avail[i].base +
6008                     (G_EXT_MEM_SIZE(hi) << 20);
6009                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6010                 i++;
6011         }
6012         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6013                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6014                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6015                 avail[i].limit = avail[i].base +
6016                     (G_EXT_MEM1_SIZE(hi) << 20);
6017                 avail[i].idx = 4;
6018                 i++;
6019         }
6020         if (!i)                                    /* no memory available */
6021                 return 0;
6022         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6023
6024         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6025         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6026         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6027         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6028         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6029         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6030         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6031         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6032         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6033
6034         /* the next few have explicit upper bounds */
6035         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6036         md->limit = md->base - 1 +
6037                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6038                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6039         md++;
6040
6041         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6042         md->limit = md->base - 1 +
6043                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6044                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6045         md++;
6046
6047         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6048                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6049                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6050                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6051         } else {
6052                 md->base = 0;
6053                 md->idx = nitems(region);  /* hide it */
6054         }
6055         md++;
6056
6057 #define ulp_region(reg) \
6058         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6059         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6060
6061         ulp_region(RX_ISCSI);
6062         ulp_region(RX_TDDP);
6063         ulp_region(TX_TPT);
6064         ulp_region(RX_STAG);
6065         ulp_region(RX_RQ);
6066         ulp_region(RX_RQUDP);
6067         ulp_region(RX_PBL);
6068         ulp_region(TX_PBL);
6069 #undef ulp_region
6070
6071         md->base = 0;
6072         md->idx = nitems(region);
6073         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6074                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6075                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6076                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6077         }
6078         md++;
6079
6080         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6081         md->limit = md->base + sc->tids.ntids - 1;
6082         md++;
6083         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6084         md->limit = md->base + sc->tids.ntids - 1;
6085         md++;
6086
6087         md->base = sc->vres.ocq.start;
6088         if (sc->vres.ocq.size)
6089                 md->limit = md->base + sc->vres.ocq.size - 1;
6090         else
6091                 md->idx = nitems(region);  /* hide it */
6092         md++;
6093
6094         /* add any address-space holes, there can be up to 3 */
6095         for (n = 0; n < i - 1; n++)
6096                 if (avail[n].limit < avail[n + 1].base)
6097                         (md++)->base = avail[n].limit;
6098         if (avail[n].limit)
6099                 (md++)->base = avail[n].limit;
6100
6101         n = md - mem;
6102         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6103
6104         for (lo = 0; lo < i; lo++)
6105                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6106                                 avail[lo].limit - 1);
6107
6108         sbuf_printf(sb, "\n");
6109         for (i = 0; i < n; i++) {
6110                 if (mem[i].idx >= nitems(region))
6111                         continue;                        /* skip holes */
6112                 if (!mem[i].limit)
6113                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6114                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6115                                 mem[i].limit);
6116         }
6117
6118         sbuf_printf(sb, "\n");
6119         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6120         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6121         mem_region_show(sb, "uP RAM:", lo, hi);
6122
6123         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6124         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6125         mem_region_show(sb, "uP Extmem2:", lo, hi);
6126
6127         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6128         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6129                    G_PMRXMAXPAGE(lo),
6130                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6131                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6132
6133         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6134         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6135         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6136                    G_PMTXMAXPAGE(lo),
6137                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6138                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6139         sbuf_printf(sb, "%u p-structs\n",
6140                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6141
6142         for (i = 0; i < 4; i++) {
6143                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6144                 if (is_t4(sc)) {
6145                         used = G_USED(lo);
6146                         alloc = G_ALLOC(lo);
6147                 } else {
6148                         used = G_T5_USED(lo);
6149                         alloc = G_T5_ALLOC(lo);
6150                 }
6151                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6152                            i, used, alloc);
6153         }
6154         for (i = 0; i < 4; i++) {
6155                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6156                 if (is_t4(sc)) {
6157                         used = G_USED(lo);
6158                         alloc = G_ALLOC(lo);
6159                 } else {
6160                         used = G_T5_USED(lo);
6161                         alloc = G_T5_ALLOC(lo);
6162                 }
6163                 sbuf_printf(sb,
6164                            "\nLoopback %d using %u pages out of %u allocated",
6165                            i, used, alloc);
6166         }
6167
6168         rc = sbuf_finish(sb);
6169         sbuf_delete(sb);
6170
6171         return (rc);
6172 }
6173
6174 static inline void
6175 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6176 {
6177         *mask = x | y;
6178         y = htobe64(y);
6179         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6180 }
6181
6182 static int
6183 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6184 {
6185         struct adapter *sc = arg1;
6186         struct sbuf *sb;
6187         int rc, i, n;
6188
6189         rc = sysctl_wire_old_buffer(req, 0);
6190         if (rc != 0)
6191                 return (rc);
6192
6193         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6194         if (sb == NULL)
6195                 return (ENOMEM);
6196
6197         sbuf_printf(sb,
6198             "Idx  Ethernet address     Mask     Vld Ports PF"
6199             "  VF              Replication             P0 P1 P2 P3  ML");
6200         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6201             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6202         for (i = 0; i < n; i++) {
6203                 uint64_t tcamx, tcamy, mask;
6204                 uint32_t cls_lo, cls_hi;
6205                 uint8_t addr[ETHER_ADDR_LEN];
6206
6207                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6208                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6209                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6210                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6211
6212                 if (tcamx & tcamy)
6213                         continue;
6214
6215                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6216                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6217                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6218                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6219                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6220                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6221                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6222
6223                 if (cls_lo & F_REPLICATE) {
6224                         struct fw_ldst_cmd ldst_cmd;
6225
6226                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6227                         ldst_cmd.op_to_addrspace =
6228                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6229                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6230                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6231                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6232                         ldst_cmd.u.mps.fid_ctl =
6233                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6234                                 V_FW_LDST_CMD_CTL(i));
6235
6236                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6237                             "t4mps");
6238                         if (rc)
6239                                 break;
6240                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6241                             sizeof(ldst_cmd), &ldst_cmd);
6242                         end_synchronized_op(sc, 0);
6243
6244                         if (rc != 0) {
6245                                 sbuf_printf(sb,
6246                                     " ------------ error %3u ------------", rc);
6247                                 rc = 0;
6248                         } else {
6249                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6250                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6251                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6252                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6253                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6254                         }
6255                 } else
6256                         sbuf_printf(sb, "%36s", "");
6257
6258                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6259                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6260                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6261         }
6262
6263         if (rc)
6264                 (void) sbuf_finish(sb);
6265         else
6266                 rc = sbuf_finish(sb);
6267         sbuf_delete(sb);
6268
6269         return (rc);
6270 }
6271
6272 static int
6273 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6274 {
6275         struct adapter *sc = arg1;
6276         struct sbuf *sb;
6277         int rc;
6278         uint16_t mtus[NMTUS];
6279
6280         rc = sysctl_wire_old_buffer(req, 0);
6281         if (rc != 0)
6282                 return (rc);
6283
6284         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6285         if (sb == NULL)
6286                 return (ENOMEM);
6287
6288         t4_read_mtu_tbl(sc, mtus, NULL);
6289
6290         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6291             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6292             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6293             mtus[14], mtus[15]);
6294
6295         rc = sbuf_finish(sb);
6296         sbuf_delete(sb);
6297
6298         return (rc);
6299 }
6300
6301 static int
6302 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6303 {
6304         struct adapter *sc = arg1;
6305         struct sbuf *sb;
6306         int rc, i;
6307         uint32_t cnt[PM_NSTATS];
6308         uint64_t cyc[PM_NSTATS];
6309         static const char *rx_stats[] = {
6310                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6311         };
6312         static const char *tx_stats[] = {
6313                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6314         };
6315
6316         rc = sysctl_wire_old_buffer(req, 0);
6317         if (rc != 0)
6318                 return (rc);
6319
6320         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6321         if (sb == NULL)
6322                 return (ENOMEM);
6323
6324         t4_pmtx_get_stats(sc, cnt, cyc);
6325         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6326         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6327                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6328                     cyc[i]);
6329
6330         t4_pmrx_get_stats(sc, cnt, cyc);
6331         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6332         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6333                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6334                     cyc[i]);
6335
6336         rc = sbuf_finish(sb);
6337         sbuf_delete(sb);
6338
6339         return (rc);
6340 }
6341
6342 static int
6343 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6344 {
6345         struct adapter *sc = arg1;
6346         struct sbuf *sb;
6347         int rc;
6348         struct tp_rdma_stats stats;
6349
6350         rc = sysctl_wire_old_buffer(req, 0);
6351         if (rc != 0)
6352                 return (rc);
6353
6354         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6355         if (sb == NULL)
6356                 return (ENOMEM);
6357
6358         t4_tp_get_rdma_stats(sc, &stats);
6359         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6360         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6361
6362         rc = sbuf_finish(sb);
6363         sbuf_delete(sb);
6364
6365         return (rc);
6366 }
6367
6368 static int
6369 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6370 {
6371         struct adapter *sc = arg1;
6372         struct sbuf *sb;
6373         int rc;
6374         struct tp_tcp_stats v4, v6;
6375
6376         rc = sysctl_wire_old_buffer(req, 0);
6377         if (rc != 0)
6378                 return (rc);
6379
6380         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6381         if (sb == NULL)
6382                 return (ENOMEM);
6383
6384         t4_tp_get_tcp_stats(sc, &v4, &v6);
6385         sbuf_printf(sb,
6386             "                                IP                 IPv6\n");
6387         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6388             v4.tcpOutRsts, v6.tcpOutRsts);
6389         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6390             v4.tcpInSegs, v6.tcpInSegs);
6391         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6392             v4.tcpOutSegs, v6.tcpOutSegs);
6393         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6394             v4.tcpRetransSegs, v6.tcpRetransSegs);
6395
6396         rc = sbuf_finish(sb);
6397         sbuf_delete(sb);
6398
6399         return (rc);
6400 }
6401
6402 static int
6403 sysctl_tids(SYSCTL_HANDLER_ARGS)
6404 {
6405         struct adapter *sc = arg1;
6406         struct sbuf *sb;
6407         int rc;
6408         struct tid_info *t = &sc->tids;
6409
6410         rc = sysctl_wire_old_buffer(req, 0);
6411         if (rc != 0)
6412                 return (rc);
6413
6414         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6415         if (sb == NULL)
6416                 return (ENOMEM);
6417
6418         if (t->natids) {
6419                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6420                     t->atids_in_use);
6421         }
6422
6423         if (t->ntids) {
6424                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6425                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6426
6427                         if (b) {
6428                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6429                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6430                                     t->ntids - 1);
6431                         } else {
6432                                 sbuf_printf(sb, "TID range: %u-%u",
6433                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6434                                     t->ntids - 1);
6435                         }
6436                 } else
6437                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6438                 sbuf_printf(sb, ", in use: %u\n",
6439                     atomic_load_acq_int(&t->tids_in_use));
6440         }
6441
6442         if (t->nstids) {
6443                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6444                     t->stid_base + t->nstids - 1, t->stids_in_use);
6445         }
6446
6447         if (t->nftids) {
6448                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6449                     t->ftid_base + t->nftids - 1);
6450         }
6451
6452         if (t->netids) {
6453                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6454                     t->etid_base + t->netids - 1);
6455         }
6456
6457         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6458             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6459             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6460
6461         rc = sbuf_finish(sb);
6462         sbuf_delete(sb);
6463
6464         return (rc);
6465 }
6466
6467 static int
6468 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6469 {
6470         struct adapter *sc = arg1;
6471         struct sbuf *sb;
6472         int rc;
6473         struct tp_err_stats stats;
6474
6475         rc = sysctl_wire_old_buffer(req, 0);
6476         if (rc != 0)
6477                 return (rc);
6478
6479         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6480         if (sb == NULL)
6481                 return (ENOMEM);
6482
6483         t4_tp_get_err_stats(sc, &stats);
6484
6485         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6486                       "channel 3\n");
6487         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6488             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6489             stats.macInErrs[3]);
6490         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6491             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6492             stats.hdrInErrs[3]);
6493         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6494             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6495             stats.tcpInErrs[3]);
6496         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6497             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6498             stats.tcp6InErrs[3]);
6499         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6500             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6501             stats.tnlCongDrops[3]);
6502         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6503             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6504             stats.tnlTxDrops[3]);
6505         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6506             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6507             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6508         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6509             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6510             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6511         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6512             stats.ofldNoNeigh, stats.ofldCongDefer);
6513
6514         rc = sbuf_finish(sb);
6515         sbuf_delete(sb);
6516
6517         return (rc);
6518 }
6519
6520 struct field_desc {
6521         const char *name;
6522         u_int start;
6523         u_int width;
6524 };
6525
6526 static void
6527 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6528 {
6529         char buf[32];
6530         int line_size = 0;
6531
6532         while (f->name) {
6533                 uint64_t mask = (1ULL << f->width) - 1;
6534                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6535                     ((uintmax_t)v >> f->start) & mask);
6536
6537                 if (line_size + len >= 79) {
6538                         line_size = 8;
6539                         sbuf_printf(sb, "\n        ");
6540                 }
6541                 sbuf_printf(sb, "%s ", buf);
6542                 line_size += len + 1;
6543                 f++;
6544         }
6545         sbuf_printf(sb, "\n");
6546 }
6547
6548 static struct field_desc tp_la0[] = {
6549         { "RcfOpCodeOut", 60, 4 },
6550         { "State", 56, 4 },
6551         { "WcfState", 52, 4 },
6552         { "RcfOpcSrcOut", 50, 2 },
6553         { "CRxError", 49, 1 },
6554         { "ERxError", 48, 1 },
6555         { "SanityFailed", 47, 1 },
6556         { "SpuriousMsg", 46, 1 },
6557         { "FlushInputMsg", 45, 1 },
6558         { "FlushInputCpl", 44, 1 },
6559         { "RssUpBit", 43, 1 },
6560         { "RssFilterHit", 42, 1 },
6561         { "Tid", 32, 10 },
6562         { "InitTcb", 31, 1 },
6563         { "LineNumber", 24, 7 },
6564         { "Emsg", 23, 1 },
6565         { "EdataOut", 22, 1 },
6566         { "Cmsg", 21, 1 },
6567         { "CdataOut", 20, 1 },
6568         { "EreadPdu", 19, 1 },
6569         { "CreadPdu", 18, 1 },
6570         { "TunnelPkt", 17, 1 },
6571         { "RcfPeerFin", 16, 1 },
6572         { "RcfReasonOut", 12, 4 },
6573         { "TxCchannel", 10, 2 },
6574         { "RcfTxChannel", 8, 2 },
6575         { "RxEchannel", 6, 2 },
6576         { "RcfRxChannel", 5, 1 },
6577         { "RcfDataOutSrdy", 4, 1 },
6578         { "RxDvld", 3, 1 },
6579         { "RxOoDvld", 2, 1 },
6580         { "RxCongestion", 1, 1 },
6581         { "TxCongestion", 0, 1 },
6582         { NULL }
6583 };
6584
6585 static struct field_desc tp_la1[] = {
6586         { "CplCmdIn", 56, 8 },
6587         { "CplCmdOut", 48, 8 },
6588         { "ESynOut", 47, 1 },
6589         { "EAckOut", 46, 1 },
6590         { "EFinOut", 45, 1 },
6591         { "ERstOut", 44, 1 },
6592         { "SynIn", 43, 1 },
6593         { "AckIn", 42, 1 },
6594         { "FinIn", 41, 1 },
6595         { "RstIn", 40, 1 },
6596         { "DataIn", 39, 1 },
6597         { "DataInVld", 38, 1 },
6598         { "PadIn", 37, 1 },
6599         { "RxBufEmpty", 36, 1 },
6600         { "RxDdp", 35, 1 },
6601         { "RxFbCongestion", 34, 1 },
6602         { "TxFbCongestion", 33, 1 },
6603         { "TxPktSumSrdy", 32, 1 },
6604         { "RcfUlpType", 28, 4 },
6605         { "Eread", 27, 1 },
6606         { "Ebypass", 26, 1 },
6607         { "Esave", 25, 1 },
6608         { "Static0", 24, 1 },
6609         { "Cread", 23, 1 },
6610         { "Cbypass", 22, 1 },
6611         { "Csave", 21, 1 },
6612         { "CPktOut", 20, 1 },
6613         { "RxPagePoolFull", 18, 2 },
6614         { "RxLpbkPkt", 17, 1 },
6615         { "TxLpbkPkt", 16, 1 },
6616         { "RxVfValid", 15, 1 },
6617         { "SynLearned", 14, 1 },
6618         { "SetDelEntry", 13, 1 },
6619         { "SetInvEntry", 12, 1 },
6620         { "CpcmdDvld", 11, 1 },
6621         { "CpcmdSave", 10, 1 },
6622         { "RxPstructsFull", 8, 2 },
6623         { "EpcmdDvld", 7, 1 },
6624         { "EpcmdFlush", 6, 1 },
6625         { "EpcmdTrimPrefix", 5, 1 },
6626         { "EpcmdTrimPostfix", 4, 1 },
6627         { "ERssIp4Pkt", 3, 1 },
6628         { "ERssIp6Pkt", 2, 1 },
6629         { "ERssTcpUdpPkt", 1, 1 },
6630         { "ERssFceFipPkt", 0, 1 },
6631         { NULL }
6632 };
6633
6634 static struct field_desc tp_la2[] = {
6635         { "CplCmdIn", 56, 8 },
6636         { "MpsVfVld", 55, 1 },
6637         { "MpsPf", 52, 3 },
6638         { "MpsVf", 44, 8 },
6639         { "SynIn", 43, 1 },
6640         { "AckIn", 42, 1 },
6641         { "FinIn", 41, 1 },
6642         { "RstIn", 40, 1 },
6643         { "DataIn", 39, 1 },
6644         { "DataInVld", 38, 1 },
6645         { "PadIn", 37, 1 },
6646         { "RxBufEmpty", 36, 1 },
6647         { "RxDdp", 35, 1 },
6648         { "RxFbCongestion", 34, 1 },
6649         { "TxFbCongestion", 33, 1 },
6650         { "TxPktSumSrdy", 32, 1 },
6651         { "RcfUlpType", 28, 4 },
6652         { "Eread", 27, 1 },
6653         { "Ebypass", 26, 1 },
6654         { "Esave", 25, 1 },
6655         { "Static0", 24, 1 },
6656         { "Cread", 23, 1 },
6657         { "Cbypass", 22, 1 },
6658         { "Csave", 21, 1 },
6659         { "CPktOut", 20, 1 },
6660         { "RxPagePoolFull", 18, 2 },
6661         { "RxLpbkPkt", 17, 1 },
6662         { "TxLpbkPkt", 16, 1 },
6663         { "RxVfValid", 15, 1 },
6664         { "SynLearned", 14, 1 },
6665         { "SetDelEntry", 13, 1 },
6666         { "SetInvEntry", 12, 1 },
6667         { "CpcmdDvld", 11, 1 },
6668         { "CpcmdSave", 10, 1 },
6669         { "RxPstructsFull", 8, 2 },
6670         { "EpcmdDvld", 7, 1 },
6671         { "EpcmdFlush", 6, 1 },
6672         { "EpcmdTrimPrefix", 5, 1 },
6673         { "EpcmdTrimPostfix", 4, 1 },
6674         { "ERssIp4Pkt", 3, 1 },
6675         { "ERssIp6Pkt", 2, 1 },
6676         { "ERssTcpUdpPkt", 1, 1 },
6677         { "ERssFceFipPkt", 0, 1 },
6678         { NULL }
6679 };
6680
6681 static void
6682 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6683 {
6684
6685         field_desc_show(sb, *p, tp_la0);
6686 }
6687
6688 static void
6689 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6690 {
6691
6692         if (idx)
6693                 sbuf_printf(sb, "\n");
6694         field_desc_show(sb, p[0], tp_la0);
6695         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6696                 field_desc_show(sb, p[1], tp_la0);
6697 }
6698
6699 static void
6700 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6701 {
6702
6703         if (idx)
6704                 sbuf_printf(sb, "\n");
6705         field_desc_show(sb, p[0], tp_la0);
6706         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6707                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6708 }
6709
6710 static int
6711 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6712 {
6713         struct adapter *sc = arg1;
6714         struct sbuf *sb;
6715         uint64_t *buf, *p;
6716         int rc;
6717         u_int i, inc;
6718         void (*show_func)(struct sbuf *, uint64_t *, int);
6719
6720         rc = sysctl_wire_old_buffer(req, 0);
6721         if (rc != 0)
6722                 return (rc);
6723
6724         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6725         if (sb == NULL)
6726                 return (ENOMEM);
6727
6728         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6729
6730         t4_tp_read_la(sc, buf, NULL);
6731         p = buf;
6732
6733         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6734         case 2:
6735                 inc = 2;
6736                 show_func = tp_la_show2;
6737                 break;
6738         case 3:
6739                 inc = 2;
6740                 show_func = tp_la_show3;
6741                 break;
6742         default:
6743                 inc = 1;
6744                 show_func = tp_la_show;
6745         }
6746
6747         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6748                 (*show_func)(sb, p, i);
6749
6750         rc = sbuf_finish(sb);
6751         sbuf_delete(sb);
6752         free(buf, M_CXGBE);
6753         return (rc);
6754 }
6755
6756 static int
6757 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6758 {
6759         struct adapter *sc = arg1;
6760         struct sbuf *sb;
6761         int rc;
6762         u64 nrate[NCHAN], orate[NCHAN];
6763
6764         rc = sysctl_wire_old_buffer(req, 0);
6765         if (rc != 0)
6766                 return (rc);
6767
6768         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6769         if (sb == NULL)
6770                 return (ENOMEM);
6771
6772         t4_get_chan_txrate(sc, nrate, orate);
6773         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6774                  "channel 3\n");
6775         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6776             nrate[0], nrate[1], nrate[2], nrate[3]);
6777         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6778             orate[0], orate[1], orate[2], orate[3]);
6779
6780         rc = sbuf_finish(sb);
6781         sbuf_delete(sb);
6782
6783         return (rc);
6784 }
6785
6786 static int
6787 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6788 {
6789         struct adapter *sc = arg1;
6790         struct sbuf *sb;
6791         uint32_t *buf, *p;
6792         int rc, i;
6793
6794         rc = sysctl_wire_old_buffer(req, 0);
6795         if (rc != 0)
6796                 return (rc);
6797
6798         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6799         if (sb == NULL)
6800                 return (ENOMEM);
6801
6802         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6803             M_ZERO | M_WAITOK);
6804
6805         t4_ulprx_read_la(sc, buf);
6806         p = buf;
6807
6808         sbuf_printf(sb, "      Pcmd        Type   Message"
6809             "                Data");
6810         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6811                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6812                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6813         }
6814
6815         rc = sbuf_finish(sb);
6816         sbuf_delete(sb);
6817         free(buf, M_CXGBE);
6818         return (rc);
6819 }
6820
6821 static int
6822 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6823 {
6824         struct adapter *sc = arg1;
6825         struct sbuf *sb;
6826         int rc, v;
6827
6828         rc = sysctl_wire_old_buffer(req, 0);
6829         if (rc != 0)
6830                 return (rc);
6831
6832         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6833         if (sb == NULL)
6834                 return (ENOMEM);
6835
6836         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6837         if (G_STATSOURCE_T5(v) == 7) {
6838                 if (G_STATMODE(v) == 0) {
6839                         sbuf_printf(sb, "total %d, incomplete %d",
6840                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6841                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6842                 } else if (G_STATMODE(v) == 1) {
6843                         sbuf_printf(sb, "total %d, data overflow %d",
6844                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6845                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6846                 }
6847         }
6848         rc = sbuf_finish(sb);
6849         sbuf_delete(sb);
6850
6851         return (rc);
6852 }
6853 #endif
6854
6855 static inline void
6856 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6857 {
6858         struct buf_ring *br;
6859         struct mbuf *m;
6860
6861         TXQ_LOCK_ASSERT_OWNED(txq);
6862
6863         br = txq->br;
6864         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6865         if (m)
6866                 t4_eth_tx(ifp, txq, m);
6867 }
6868
6869 void
6870 t4_tx_callout(void *arg)
6871 {
6872         struct sge_eq *eq = arg;
6873         struct adapter *sc;
6874
6875         if (EQ_TRYLOCK(eq) == 0)
6876                 goto reschedule;
6877
6878         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6879                 EQ_UNLOCK(eq);
6880 reschedule:
6881                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6882                         callout_schedule(&eq->tx_callout, 1);
6883                 return;
6884         }
6885
6886         EQ_LOCK_ASSERT_OWNED(eq);
6887
6888         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6889
6890                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6891                         struct sge_txq *txq = arg;
6892                         struct port_info *pi = txq->ifp->if_softc;
6893
6894                         sc = pi->adapter;
6895                 } else {
6896                         struct sge_wrq *wrq = arg;
6897
6898                         sc = wrq->adapter;
6899                 }
6900
6901                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6902         }
6903
6904         EQ_UNLOCK(eq);
6905 }
6906
6907 void
6908 t4_tx_task(void *arg, int count)
6909 {
6910         struct sge_eq *eq = arg;
6911
6912         EQ_LOCK(eq);
6913         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6914                 struct sge_txq *txq = arg;
6915                 txq_start(txq->ifp, txq);
6916         } else {
6917                 struct sge_wrq *wrq = arg;
6918                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6919         }
6920         EQ_UNLOCK(eq);
6921 }
6922
6923 static uint32_t
6924 fconf_to_mode(uint32_t fconf)
6925 {
6926         uint32_t mode;
6927
6928         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6929             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6930
6931         if (fconf & F_FRAGMENTATION)
6932                 mode |= T4_FILTER_IP_FRAGMENT;
6933
6934         if (fconf & F_MPSHITTYPE)
6935                 mode |= T4_FILTER_MPS_HIT_TYPE;
6936
6937         if (fconf & F_MACMATCH)
6938                 mode |= T4_FILTER_MAC_IDX;
6939
6940         if (fconf & F_ETHERTYPE)
6941                 mode |= T4_FILTER_ETH_TYPE;
6942
6943         if (fconf & F_PROTOCOL)
6944                 mode |= T4_FILTER_IP_PROTO;
6945
6946         if (fconf & F_TOS)
6947                 mode |= T4_FILTER_IP_TOS;
6948
6949         if (fconf & F_VLAN)
6950                 mode |= T4_FILTER_VLAN;
6951
6952         if (fconf & F_VNIC_ID)
6953                 mode |= T4_FILTER_VNIC;
6954
6955         if (fconf & F_PORT)
6956                 mode |= T4_FILTER_PORT;
6957
6958         if (fconf & F_FCOE)
6959                 mode |= T4_FILTER_FCoE;
6960
6961         return (mode);
6962 }
6963
6964 static uint32_t
6965 mode_to_fconf(uint32_t mode)
6966 {
6967         uint32_t fconf = 0;
6968
6969         if (mode & T4_FILTER_IP_FRAGMENT)
6970                 fconf |= F_FRAGMENTATION;
6971
6972         if (mode & T4_FILTER_MPS_HIT_TYPE)
6973                 fconf |= F_MPSHITTYPE;
6974
6975         if (mode & T4_FILTER_MAC_IDX)
6976                 fconf |= F_MACMATCH;
6977
6978         if (mode & T4_FILTER_ETH_TYPE)
6979                 fconf |= F_ETHERTYPE;
6980
6981         if (mode & T4_FILTER_IP_PROTO)
6982                 fconf |= F_PROTOCOL;
6983
6984         if (mode & T4_FILTER_IP_TOS)
6985                 fconf |= F_TOS;
6986
6987         if (mode & T4_FILTER_VLAN)
6988                 fconf |= F_VLAN;
6989
6990         if (mode & T4_FILTER_VNIC)
6991                 fconf |= F_VNIC_ID;
6992
6993         if (mode & T4_FILTER_PORT)
6994                 fconf |= F_PORT;
6995
6996         if (mode & T4_FILTER_FCoE)
6997                 fconf |= F_FCOE;
6998
6999         return (fconf);
7000 }
7001
7002 static uint32_t
7003 fspec_to_fconf(struct t4_filter_specification *fs)
7004 {
7005         uint32_t fconf = 0;
7006
7007         if (fs->val.frag || fs->mask.frag)
7008                 fconf |= F_FRAGMENTATION;
7009
7010         if (fs->val.matchtype || fs->mask.matchtype)
7011                 fconf |= F_MPSHITTYPE;
7012
7013         if (fs->val.macidx || fs->mask.macidx)
7014                 fconf |= F_MACMATCH;
7015
7016         if (fs->val.ethtype || fs->mask.ethtype)
7017                 fconf |= F_ETHERTYPE;
7018
7019         if (fs->val.proto || fs->mask.proto)
7020                 fconf |= F_PROTOCOL;
7021
7022         if (fs->val.tos || fs->mask.tos)
7023                 fconf |= F_TOS;
7024
7025         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7026                 fconf |= F_VLAN;
7027
7028         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7029                 fconf |= F_VNIC_ID;
7030
7031         if (fs->val.iport || fs->mask.iport)
7032                 fconf |= F_PORT;
7033
7034         if (fs->val.fcoe || fs->mask.fcoe)
7035                 fconf |= F_FCOE;
7036
7037         return (fconf);
7038 }
7039
7040 static int
7041 get_filter_mode(struct adapter *sc, uint32_t *mode)
7042 {
7043         int rc;
7044         uint32_t fconf;
7045
7046         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7047             "t4getfm");
7048         if (rc)
7049                 return (rc);
7050
7051         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7052             A_TP_VLAN_PRI_MAP);
7053
7054         if (sc->params.tp.vlan_pri_map != fconf) {
7055                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7056                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7057                     fconf);
7058                 sc->params.tp.vlan_pri_map = fconf;
7059         }
7060
7061         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7062
7063         end_synchronized_op(sc, LOCK_HELD);
7064         return (0);
7065 }
7066
7067 static int
7068 set_filter_mode(struct adapter *sc, uint32_t mode)
7069 {
7070         uint32_t fconf;
7071         int rc;
7072
7073         fconf = mode_to_fconf(mode);
7074
7075         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7076             "t4setfm");
7077         if (rc)
7078                 return (rc);
7079
7080         if (sc->tids.ftids_in_use > 0) {
7081                 rc = EBUSY;
7082                 goto done;
7083         }
7084
7085 #ifdef TCP_OFFLOAD
7086         if (sc->offload_map) {
7087                 rc = EBUSY;
7088                 goto done;
7089         }
7090 #endif
7091
7092 #ifdef notyet
7093         rc = -t4_set_filter_mode(sc, fconf);
7094         if (rc == 0)
7095                 sc->filter_mode = fconf;
7096 #else
7097         rc = ENOTSUP;
7098 #endif
7099
7100 done:
7101         end_synchronized_op(sc, LOCK_HELD);
7102         return (rc);
7103 }
7104
7105 static inline uint64_t
7106 get_filter_hits(struct adapter *sc, uint32_t fid)
7107 {
7108         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7109         uint64_t hits;
7110
7111         memwin_info(sc, 0, &mw_base, NULL);
7112         off = position_memwin(sc, 0,
7113             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7114         if (is_t4(sc)) {
7115                 hits = t4_read_reg64(sc, mw_base + off + 16);
7116                 hits = be64toh(hits);
7117         } else {
7118                 hits = t4_read_reg(sc, mw_base + off + 24);
7119                 hits = be32toh(hits);
7120         }
7121
7122         return (hits);
7123 }
7124
7125 static int
7126 get_filter(struct adapter *sc, struct t4_filter *t)
7127 {
7128         int i, rc, nfilters = sc->tids.nftids;
7129         struct filter_entry *f;
7130
7131         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7132             "t4getf");
7133         if (rc)
7134                 return (rc);
7135
7136         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7137             t->idx >= nfilters) {
7138                 t->idx = 0xffffffff;
7139                 goto done;
7140         }
7141
7142         f = &sc->tids.ftid_tab[t->idx];
7143         for (i = t->idx; i < nfilters; i++, f++) {
7144                 if (f->valid) {
7145                         t->idx = i;
7146                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7147                         t->smtidx = f->smtidx;
7148                         if (f->fs.hitcnts)
7149                                 t->hits = get_filter_hits(sc, t->idx);
7150                         else
7151                                 t->hits = UINT64_MAX;
7152                         t->fs = f->fs;
7153
7154                         goto done;
7155                 }
7156         }
7157
7158         t->idx = 0xffffffff;
7159 done:
7160         end_synchronized_op(sc, LOCK_HELD);
7161         return (0);
7162 }
7163
7164 static int
7165 set_filter(struct adapter *sc, struct t4_filter *t)
7166 {
7167         unsigned int nfilters, nports;
7168         struct filter_entry *f;
7169         int i, rc;
7170
7171         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7172         if (rc)
7173                 return (rc);
7174
7175         nfilters = sc->tids.nftids;
7176         nports = sc->params.nports;
7177
7178         if (nfilters == 0) {
7179                 rc = ENOTSUP;
7180                 goto done;
7181         }
7182
7183         if (!(sc->flags & FULL_INIT_DONE)) {
7184                 rc = EAGAIN;
7185                 goto done;
7186         }
7187
7188         if (t->idx >= nfilters) {
7189                 rc = EINVAL;
7190                 goto done;
7191         }
7192
7193         /* Validate against the global filter mode */
7194         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7195             sc->params.tp.vlan_pri_map) {
7196                 rc = E2BIG;
7197                 goto done;
7198         }
7199
7200         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7201                 rc = EINVAL;
7202                 goto done;
7203         }
7204
7205         if (t->fs.val.iport >= nports) {
7206                 rc = EINVAL;
7207                 goto done;
7208         }
7209
7210         /* Can't specify an iq if not steering to it */
7211         if (!t->fs.dirsteer && t->fs.iq) {
7212                 rc = EINVAL;
7213                 goto done;
7214         }
7215
7216         /* IPv6 filter idx must be 4 aligned */
7217         if (t->fs.type == 1 &&
7218             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7219                 rc = EINVAL;
7220                 goto done;
7221         }
7222
7223         if (sc->tids.ftid_tab == NULL) {
7224                 KASSERT(sc->tids.ftids_in_use == 0,
7225                     ("%s: no memory allocated but filters_in_use > 0",
7226                     __func__));
7227
7228                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7229                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7230                 if (sc->tids.ftid_tab == NULL) {
7231                         rc = ENOMEM;
7232                         goto done;
7233                 }
7234                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7235         }
7236
7237         for (i = 0; i < 4; i++) {
7238                 f = &sc->tids.ftid_tab[t->idx + i];
7239
7240                 if (f->pending || f->valid) {
7241                         rc = EBUSY;
7242                         goto done;
7243                 }
7244                 if (f->locked) {
7245                         rc = EPERM;
7246                         goto done;
7247                 }
7248
7249                 if (t->fs.type == 0)
7250                         break;
7251         }
7252
7253         f = &sc->tids.ftid_tab[t->idx];
7254         f->fs = t->fs;
7255
7256         rc = set_filter_wr(sc, t->idx);
7257 done:
7258         end_synchronized_op(sc, 0);
7259
7260         if (rc == 0) {
7261                 mtx_lock(&sc->tids.ftid_lock);
7262                 for (;;) {
7263                         if (f->pending == 0) {
7264                                 rc = f->valid ? 0 : EIO;
7265                                 break;
7266                         }
7267
7268                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7269                             PCATCH, "t4setfw", 0)) {
7270                                 rc = EINPROGRESS;
7271                                 break;
7272                         }
7273                 }
7274                 mtx_unlock(&sc->tids.ftid_lock);
7275         }
7276         return (rc);
7277 }
7278
7279 static int
7280 del_filter(struct adapter *sc, struct t4_filter *t)
7281 {
7282         unsigned int nfilters;
7283         struct filter_entry *f;
7284         int rc;
7285
7286         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7287         if (rc)
7288                 return (rc);
7289
7290         nfilters = sc->tids.nftids;
7291
7292         if (nfilters == 0) {
7293                 rc = ENOTSUP;
7294                 goto done;
7295         }
7296
7297         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7298             t->idx >= nfilters) {
7299                 rc = EINVAL;
7300                 goto done;
7301         }
7302
7303         if (!(sc->flags & FULL_INIT_DONE)) {
7304                 rc = EAGAIN;
7305                 goto done;
7306         }
7307
7308         f = &sc->tids.ftid_tab[t->idx];
7309
7310         if (f->pending) {
7311                 rc = EBUSY;
7312                 goto done;
7313         }
7314         if (f->locked) {
7315                 rc = EPERM;
7316                 goto done;
7317         }
7318
7319         if (f->valid) {
7320                 t->fs = f->fs;  /* extra info for the caller */
7321                 rc = del_filter_wr(sc, t->idx);
7322         }
7323
7324 done:
7325         end_synchronized_op(sc, 0);
7326
7327         if (rc == 0) {
7328                 mtx_lock(&sc->tids.ftid_lock);
7329                 for (;;) {
7330                         if (f->pending == 0) {
7331                                 rc = f->valid ? EIO : 0;
7332                                 break;
7333                         }
7334
7335                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7336                             PCATCH, "t4delfw", 0)) {
7337                                 rc = EINPROGRESS;
7338                                 break;
7339                         }
7340                 }
7341                 mtx_unlock(&sc->tids.ftid_lock);
7342         }
7343
7344         return (rc);
7345 }
7346
7347 static void
7348 clear_filter(struct filter_entry *f)
7349 {
7350         if (f->l2t)
7351                 t4_l2t_release(f->l2t);
7352
7353         bzero(f, sizeof (*f));
7354 }
7355
7356 static int
7357 set_filter_wr(struct adapter *sc, int fidx)
7358 {
7359         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7360         struct wrqe *wr;
7361         struct fw_filter_wr *fwr;
7362         unsigned int ftid;
7363
7364         ASSERT_SYNCHRONIZED_OP(sc);
7365
7366         if (f->fs.newdmac || f->fs.newvlan) {
7367                 /* This filter needs an L2T entry; allocate one. */
7368                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7369                 if (f->l2t == NULL)
7370                         return (EAGAIN);
7371                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7372                     f->fs.dmac)) {
7373                         t4_l2t_release(f->l2t);
7374                         f->l2t = NULL;
7375                         return (ENOMEM);
7376                 }
7377         }
7378
7379         ftid = sc->tids.ftid_base + fidx;
7380
7381         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7382         if (wr == NULL)
7383                 return (ENOMEM);
7384
7385         fwr = wrtod(wr);
7386         bzero(fwr, sizeof (*fwr));
7387
7388         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7389         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7390         fwr->tid_to_iq =
7391             htobe32(V_FW_FILTER_WR_TID(ftid) |
7392                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7393                 V_FW_FILTER_WR_NOREPLY(0) |
7394                 V_FW_FILTER_WR_IQ(f->fs.iq));
7395         fwr->del_filter_to_l2tix =
7396             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7397                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7398                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7399                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7400                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7401                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7402                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7403                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7404                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7405                     f->fs.newvlan == VLAN_REWRITE) |
7406                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7407                     f->fs.newvlan == VLAN_REWRITE) |
7408                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7409                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7410                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7411                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7412         fwr->ethtype = htobe16(f->fs.val.ethtype);
7413         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7414         fwr->frag_to_ovlan_vldm =
7415             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7416                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7417                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7418                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7419                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7420                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7421         fwr->smac_sel = 0;
7422         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7423             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7424         fwr->maci_to_matchtypem =
7425             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7426                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7427                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7428                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7429                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7430                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7431                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7432                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7433         fwr->ptcl = f->fs.val.proto;
7434         fwr->ptclm = f->fs.mask.proto;
7435         fwr->ttyp = f->fs.val.tos;
7436         fwr->ttypm = f->fs.mask.tos;
7437         fwr->ivlan = htobe16(f->fs.val.vlan);
7438         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7439         fwr->ovlan = htobe16(f->fs.val.vnic);
7440         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7441         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7442         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7443         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7444         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7445         fwr->lp = htobe16(f->fs.val.dport);
7446         fwr->lpm = htobe16(f->fs.mask.dport);
7447         fwr->fp = htobe16(f->fs.val.sport);
7448         fwr->fpm = htobe16(f->fs.mask.sport);
7449         if (f->fs.newsmac)
7450                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7451
7452         f->pending = 1;
7453         sc->tids.ftids_in_use++;
7454
7455         t4_wrq_tx(sc, wr);
7456         return (0);
7457 }
7458
7459 static int
7460 del_filter_wr(struct adapter *sc, int fidx)
7461 {
7462         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7463         struct wrqe *wr;
7464         struct fw_filter_wr *fwr;
7465         unsigned int ftid;
7466
7467         ftid = sc->tids.ftid_base + fidx;
7468
7469         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7470         if (wr == NULL)
7471                 return (ENOMEM);
7472         fwr = wrtod(wr);
7473         bzero(fwr, sizeof (*fwr));
7474
7475         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7476
7477         f->pending = 1;
7478         t4_wrq_tx(sc, wr);
7479         return (0);
7480 }
7481
7482 int
7483 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7484 {
7485         struct adapter *sc = iq->adapter;
7486         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7487         unsigned int idx = GET_TID(rpl);
7488         unsigned int rc;
7489         struct filter_entry *f;
7490
7491         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7492             rss->opcode));
7493
7494         if (is_ftid(sc, idx)) {
7495
7496                 idx -= sc->tids.ftid_base;
7497                 f = &sc->tids.ftid_tab[idx];
7498                 rc = G_COOKIE(rpl->cookie);
7499
7500                 mtx_lock(&sc->tids.ftid_lock);
7501                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7502                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7503                             __func__, idx));
7504                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7505                         f->pending = 0;  /* asynchronous setup completed */
7506                         f->valid = 1;
7507                 } else {
7508                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7509                                 /* Add or delete failed, display an error */
7510                                 log(LOG_ERR,
7511                                     "filter %u setup failed with error %u\n",
7512                                     idx, rc);
7513                         }
7514
7515                         clear_filter(f);
7516                         sc->tids.ftids_in_use--;
7517                 }
7518                 wakeup(&sc->tids.ftid_tab);
7519                 mtx_unlock(&sc->tids.ftid_lock);
7520         }
7521
7522         return (0);
7523 }
7524
7525 static int
7526 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7527 {
7528         int rc;
7529
7530         if (cntxt->cid > M_CTXTQID)
7531                 return (EINVAL);
7532
7533         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7534             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7535                 return (EINVAL);
7536
7537         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7538         if (rc)
7539                 return (rc);
7540
7541         if (sc->flags & FW_OK) {
7542                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7543                     &cntxt->data[0]);
7544                 if (rc == 0)
7545                         goto done;
7546         }
7547
7548         /*
7549          * Read via firmware failed or wasn't even attempted.  Read directly via
7550          * the backdoor.
7551          */
7552         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7553 done:
7554         end_synchronized_op(sc, 0);
7555         return (rc);
7556 }
7557
7558 static int
7559 load_fw(struct adapter *sc, struct t4_data *fw)
7560 {
7561         int rc;
7562         uint8_t *fw_data;
7563
7564         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7565         if (rc)
7566                 return (rc);
7567
7568         if (sc->flags & FULL_INIT_DONE) {
7569                 rc = EBUSY;
7570                 goto done;
7571         }
7572
7573         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7574         if (fw_data == NULL) {
7575                 rc = ENOMEM;
7576                 goto done;
7577         }
7578
7579         rc = copyin(fw->data, fw_data, fw->len);
7580         if (rc == 0)
7581                 rc = -t4_load_fw(sc, fw_data, fw->len);
7582
7583         free(fw_data, M_CXGBE);
7584 done:
7585         end_synchronized_op(sc, 0);
7586         return (rc);
7587 }
7588
7589 static int
7590 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7591 {
7592         uint32_t addr, off, remaining, i, n;
7593         uint32_t *buf, *b;
7594         uint32_t mw_base, mw_aperture;
7595         int rc;
7596         uint8_t *dst;
7597
7598         rc = validate_mem_range(sc, mr->addr, mr->len);
7599         if (rc != 0)
7600                 return (rc);
7601
7602         memwin_info(sc, win, &mw_base, &mw_aperture);
7603         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7604         addr = mr->addr;
7605         remaining = mr->len;
7606         dst = (void *)mr->data;
7607
7608         while (remaining) {
7609                 off = position_memwin(sc, win, addr);
7610
7611                 /* number of bytes that we'll copy in the inner loop */
7612                 n = min(remaining, mw_aperture - off);
7613                 for (i = 0; i < n; i += 4)
7614                         *b++ = t4_read_reg(sc, mw_base + off + i);
7615
7616                 rc = copyout(buf, dst, n);
7617                 if (rc != 0)
7618                         break;
7619
7620                 b = buf;
7621                 dst += n;
7622                 remaining -= n;
7623                 addr += n;
7624         }
7625
7626         free(buf, M_CXGBE);
7627         return (rc);
7628 }
7629
7630 static int
7631 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7632 {
7633         int rc;
7634
7635         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7636                 return (EINVAL);
7637
7638         if (i2cd->len > sizeof(i2cd->data))
7639                 return (EFBIG);
7640
7641         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7642         if (rc)
7643                 return (rc);
7644         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7645             i2cd->offset, i2cd->len, &i2cd->data[0]);
7646         end_synchronized_op(sc, 0);
7647
7648         return (rc);
7649 }
7650
7651 static int
7652 in_range(int val, int lo, int hi)
7653 {
7654
7655         return (val < 0 || (val <= hi && val >= lo));
7656 }
7657
7658 static int
7659 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7660 {
7661         int fw_subcmd, fw_type, rc;
7662
7663         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7664         if (rc)
7665                 return (rc);
7666
7667         if (!(sc->flags & FULL_INIT_DONE)) {
7668                 rc = EAGAIN;
7669                 goto done;
7670         }
7671
7672         /*
7673          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7674          * sub-command and type are in common locations.)
7675          */
7676         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7677                 fw_subcmd = FW_SCHED_SC_CONFIG;
7678         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7679                 fw_subcmd = FW_SCHED_SC_PARAMS;
7680         else {
7681                 rc = EINVAL;
7682                 goto done;
7683         }
7684         if (p->type == SCHED_CLASS_TYPE_PACKET)
7685                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7686         else {
7687                 rc = EINVAL;
7688                 goto done;
7689         }
7690
7691         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7692                 /* Vet our parameters ..*/
7693                 if (p->u.config.minmax < 0) {
7694                         rc = EINVAL;
7695                         goto done;
7696                 }
7697
7698                 /* And pass the request to the firmware ...*/
7699                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7700                 goto done;
7701         }
7702
7703         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7704                 int fw_level;
7705                 int fw_mode;
7706                 int fw_rateunit;
7707                 int fw_ratemode;
7708
7709                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7710                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7711                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7712                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7713                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7714                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7715                 else {
7716                         rc = EINVAL;
7717                         goto done;
7718                 }
7719
7720                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7721                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7722                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7723                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7724                 else {
7725                         rc = EINVAL;
7726                         goto done;
7727                 }
7728
7729                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7730                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7731                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7732                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7733                 else {
7734                         rc = EINVAL;
7735                         goto done;
7736                 }
7737
7738                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7739                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7740                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7741                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7742                 else {
7743                         rc = EINVAL;
7744                         goto done;
7745                 }
7746
7747                 /* Vet our parameters ... */
7748                 if (!in_range(p->u.params.channel, 0, 3) ||
7749                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7750                     !in_range(p->u.params.minrate, 0, 10000000) ||
7751                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7752                     !in_range(p->u.params.weight, 0, 100)) {
7753                         rc = ERANGE;
7754                         goto done;
7755                 }
7756
7757                 /*
7758                  * Translate any unset parameters into the firmware's
7759                  * nomenclature and/or fail the call if the parameters
7760                  * are required ...
7761                  */
7762                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7763                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7764                         rc = EINVAL;
7765                         goto done;
7766                 }
7767                 if (p->u.params.minrate < 0)
7768                         p->u.params.minrate = 0;
7769                 if (p->u.params.maxrate < 0) {
7770                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7771                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7772                                 rc = EINVAL;
7773                                 goto done;
7774                         } else
7775                                 p->u.params.maxrate = 0;
7776                 }
7777                 if (p->u.params.weight < 0) {
7778                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7779                                 rc = EINVAL;
7780                                 goto done;
7781                         } else
7782                                 p->u.params.weight = 0;
7783                 }
7784                 if (p->u.params.pktsize < 0) {
7785                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7786                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7787                                 rc = EINVAL;
7788                                 goto done;
7789                         } else
7790                                 p->u.params.pktsize = 0;
7791                 }
7792
7793                 /* See what the firmware thinks of the request ... */
7794                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7795                     fw_rateunit, fw_ratemode, p->u.params.channel,
7796                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7797                     p->u.params.weight, p->u.params.pktsize, 1);
7798                 goto done;
7799         }
7800
7801         rc = EINVAL;
7802 done:
7803         end_synchronized_op(sc, 0);
7804         return (rc);
7805 }
7806
7807 static int
7808 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7809 {
7810         struct port_info *pi = NULL;
7811         struct sge_txq *txq;
7812         uint32_t fw_mnem, fw_queue, fw_class;
7813         int i, rc;
7814
7815         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7816         if (rc)
7817                 return (rc);
7818
7819         if (!(sc->flags & FULL_INIT_DONE)) {
7820                 rc = EAGAIN;
7821                 goto done;
7822         }
7823
7824         if (p->port >= sc->params.nports) {
7825                 rc = EINVAL;
7826                 goto done;
7827         }
7828
7829         pi = sc->port[p->port];
7830         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7831                 rc = EINVAL;
7832                 goto done;
7833         }
7834
7835         /*
7836          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7837          * Scheduling Class in this case).
7838          */
7839         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7840             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7841         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7842
7843         /*
7844          * If op.queue is non-negative, then we're only changing the scheduling
7845          * on a single specified TX queue.
7846          */
7847         if (p->queue >= 0) {
7848                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7849                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7850                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7851                     &fw_class);
7852                 goto done;
7853         }
7854
7855         /*
7856          * Change the scheduling on all the TX queues for the
7857          * interface.
7858          */
7859         for_each_txq(pi, i, txq) {
7860                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7861                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7862                     &fw_class);
7863                 if (rc)
7864                         goto done;
7865         }
7866
7867         rc = 0;
7868 done:
7869         end_synchronized_op(sc, 0);
7870         return (rc);
7871 }
7872
7873 int
7874 t4_os_find_pci_capability(struct adapter *sc, int cap)
7875 {
7876         int i;
7877
7878         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7879 }
7880
7881 int
7882 t4_os_pci_save_state(struct adapter *sc)
7883 {
7884         device_t dev;
7885         struct pci_devinfo *dinfo;
7886
7887         dev = sc->dev;
7888         dinfo = device_get_ivars(dev);
7889
7890         pci_cfg_save(dev, dinfo, 0);
7891         return (0);
7892 }
7893
7894 int
7895 t4_os_pci_restore_state(struct adapter *sc)
7896 {
7897         device_t dev;
7898         struct pci_devinfo *dinfo;
7899
7900         dev = sc->dev;
7901         dinfo = device_get_ivars(dev);
7902
7903         pci_cfg_restore(dev, dinfo);
7904         return (0);
7905 }
7906
7907 void
7908 t4_os_portmod_changed(const struct adapter *sc, int idx)
7909 {
7910         struct port_info *pi = sc->port[idx];
7911         static const char *mod_str[] = {
7912                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7913         };
7914
7915         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7916                 if_printf(pi->ifp, "transceiver unplugged.\n");
7917         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7918                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7919         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7920                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7921         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7922                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7923                     mod_str[pi->mod_type]);
7924         } else {
7925                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7926                     pi->mod_type);
7927         }
7928 }
7929
7930 void
7931 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7932 {
7933         struct port_info *pi = sc->port[idx];
7934         struct ifnet *ifp = pi->ifp;
7935
7936         if (link_stat) {
7937                 pi->linkdnrc = -1;
7938                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7939                 if_link_state_change(ifp, LINK_STATE_UP);
7940         } else {
7941                 if (reason >= 0)
7942                         pi->linkdnrc = reason;
7943                 if_link_state_change(ifp, LINK_STATE_DOWN);
7944         }
7945 }
7946
7947 void
7948 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7949 {
7950         struct adapter *sc;
7951
7952         sx_slock(&t4_list_lock);
7953         SLIST_FOREACH(sc, &t4_list, link) {
7954                 /*
7955                  * func should not make any assumptions about what state sc is
7956                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7957                  */
7958                 func(sc, arg);
7959         }
7960         sx_sunlock(&t4_list_lock);
7961 }
7962
7963 static int
7964 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7965 {
7966        return (0);
7967 }
7968
7969 static int
7970 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7971 {
7972        return (0);
7973 }
7974
7975 static int
7976 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7977     struct thread *td)
7978 {
7979         int rc;
7980         struct adapter *sc = dev->si_drv1;
7981
7982         rc = priv_check(td, PRIV_DRIVER);
7983         if (rc != 0)
7984                 return (rc);
7985
7986         switch (cmd) {
7987         case CHELSIO_T4_GETREG: {
7988                 struct t4_reg *edata = (struct t4_reg *)data;
7989
7990                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7991                         return (EFAULT);
7992
7993                 if (edata->size == 4)
7994                         edata->val = t4_read_reg(sc, edata->addr);
7995                 else if (edata->size == 8)
7996                         edata->val = t4_read_reg64(sc, edata->addr);
7997                 else
7998                         return (EINVAL);
7999
8000                 break;
8001         }
8002         case CHELSIO_T4_SETREG: {
8003                 struct t4_reg *edata = (struct t4_reg *)data;
8004
8005                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8006                         return (EFAULT);
8007
8008                 if (edata->size == 4) {
8009                         if (edata->val & 0xffffffff00000000)
8010                                 return (EINVAL);
8011                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8012                 } else if (edata->size == 8)
8013                         t4_write_reg64(sc, edata->addr, edata->val);
8014                 else
8015                         return (EINVAL);
8016                 break;
8017         }
8018         case CHELSIO_T4_REGDUMP: {
8019                 struct t4_regdump *regs = (struct t4_regdump *)data;
8020                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8021                 uint8_t *buf;
8022
8023                 if (regs->len < reglen) {
8024                         regs->len = reglen; /* hint to the caller */
8025                         return (ENOBUFS);
8026                 }
8027
8028                 regs->len = reglen;
8029                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8030                 t4_get_regs(sc, regs, buf);
8031                 rc = copyout(buf, regs->data, reglen);
8032                 free(buf, M_CXGBE);
8033                 break;
8034         }
8035         case CHELSIO_T4_GET_FILTER_MODE:
8036                 rc = get_filter_mode(sc, (uint32_t *)data);
8037                 break;
8038         case CHELSIO_T4_SET_FILTER_MODE:
8039                 rc = set_filter_mode(sc, *(uint32_t *)data);
8040                 break;
8041         case CHELSIO_T4_GET_FILTER:
8042                 rc = get_filter(sc, (struct t4_filter *)data);
8043                 break;
8044         case CHELSIO_T4_SET_FILTER:
8045                 rc = set_filter(sc, (struct t4_filter *)data);
8046                 break;
8047         case CHELSIO_T4_DEL_FILTER:
8048                 rc = del_filter(sc, (struct t4_filter *)data);
8049                 break;
8050         case CHELSIO_T4_GET_SGE_CONTEXT:
8051                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8052                 break;
8053         case CHELSIO_T4_LOAD_FW:
8054                 rc = load_fw(sc, (struct t4_data *)data);
8055                 break;
8056         case CHELSIO_T4_GET_MEM:
8057                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8058                 break;
8059         case CHELSIO_T4_GET_I2C:
8060                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8061                 break;
8062         case CHELSIO_T4_CLEAR_STATS: {
8063                 int i;
8064                 u_int port_id = *(uint32_t *)data;
8065                 struct port_info *pi;
8066
8067                 if (port_id >= sc->params.nports)
8068                         return (EINVAL);
8069                 pi = sc->port[port_id];
8070
8071                 /* MAC stats */
8072                 t4_clr_port_stats(sc, pi->tx_chan);
8073
8074                 if (pi->flags & PORT_INIT_DONE) {
8075                         struct sge_rxq *rxq;
8076                         struct sge_txq *txq;
8077                         struct sge_wrq *wrq;
8078
8079                         for_each_rxq(pi, i, rxq) {
8080 #if defined(INET) || defined(INET6)
8081                                 rxq->lro.lro_queued = 0;
8082                                 rxq->lro.lro_flushed = 0;
8083 #endif
8084                                 rxq->rxcsum = 0;
8085                                 rxq->vlan_extraction = 0;
8086                         }
8087
8088                         for_each_txq(pi, i, txq) {
8089                                 txq->txcsum = 0;
8090                                 txq->tso_wrs = 0;
8091                                 txq->vlan_insertion = 0;
8092                                 txq->imm_wrs = 0;
8093                                 txq->sgl_wrs = 0;
8094                                 txq->txpkt_wrs = 0;
8095                                 txq->txpkts_wrs = 0;
8096                                 txq->txpkts_pkts = 0;
8097                                 txq->br->br_drops = 0;
8098                                 txq->no_dmamap = 0;
8099                                 txq->no_desc = 0;
8100                         }
8101
8102 #ifdef TCP_OFFLOAD
8103                         /* nothing to clear for each ofld_rxq */
8104
8105                         for_each_ofld_txq(pi, i, wrq) {
8106                                 wrq->tx_wrs = 0;
8107                                 wrq->no_desc = 0;
8108                         }
8109 #endif
8110                         wrq = &sc->sge.ctrlq[pi->port_id];
8111                         wrq->tx_wrs = 0;
8112                         wrq->no_desc = 0;
8113                 }
8114                 break;
8115         }
8116         case CHELSIO_T4_SCHED_CLASS:
8117                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8118                 break;
8119         case CHELSIO_T4_SCHED_QUEUE:
8120                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8121                 break;
8122         case CHELSIO_T4_GET_TRACER:
8123                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8124                 break;
8125         case CHELSIO_T4_SET_TRACER:
8126                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8127                 break;
8128         default:
8129                 rc = EINVAL;
8130         }
8131
8132         return (rc);
8133 }
8134
8135 #ifdef TCP_OFFLOAD
8136 void
8137 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8138     const unsigned int *pgsz_order)
8139 {
8140         struct port_info *pi = ifp->if_softc;
8141         struct adapter *sc = pi->adapter;
8142
8143         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8144         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8145                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8146                 V_HPZ3(pgsz_order[3]));
8147 }
8148
8149 static int
8150 toe_capability(struct port_info *pi, int enable)
8151 {
8152         int rc;
8153         struct adapter *sc = pi->adapter;
8154
8155         ASSERT_SYNCHRONIZED_OP(sc);
8156
8157         if (!is_offload(sc))
8158                 return (ENODEV);
8159
8160         if (enable) {
8161                 if (!(sc->flags & FULL_INIT_DONE)) {
8162                         rc = cxgbe_init_synchronized(pi);
8163                         if (rc)
8164                                 return (rc);
8165                 }
8166
8167                 if (isset(&sc->offload_map, pi->port_id))
8168                         return (0);
8169
8170                 if (!(sc->flags & TOM_INIT_DONE)) {
8171                         rc = t4_activate_uld(sc, ULD_TOM);
8172                         if (rc == EAGAIN) {
8173                                 log(LOG_WARNING,
8174                                     "You must kldload t4_tom.ko before trying "
8175                                     "to enable TOE on a cxgbe interface.\n");
8176                         }
8177                         if (rc != 0)
8178                                 return (rc);
8179                         KASSERT(sc->tom_softc != NULL,
8180                             ("%s: TOM activated but softc NULL", __func__));
8181                         KASSERT(sc->flags & TOM_INIT_DONE,
8182                             ("%s: TOM activated but flag not set", __func__));
8183                 }
8184
8185                 setbit(&sc->offload_map, pi->port_id);
8186         } else {
8187                 if (!isset(&sc->offload_map, pi->port_id))
8188                         return (0);
8189
8190                 KASSERT(sc->flags & TOM_INIT_DONE,
8191                     ("%s: TOM never initialized?", __func__));
8192                 clrbit(&sc->offload_map, pi->port_id);
8193         }
8194
8195         return (0);
8196 }
8197
8198 /*
8199  * Add an upper layer driver to the global list.
8200  */
8201 int
8202 t4_register_uld(struct uld_info *ui)
8203 {
8204         int rc = 0;
8205         struct uld_info *u;
8206
8207         sx_xlock(&t4_uld_list_lock);
8208         SLIST_FOREACH(u, &t4_uld_list, link) {
8209             if (u->uld_id == ui->uld_id) {
8210                     rc = EEXIST;
8211                     goto done;
8212             }
8213         }
8214
8215         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8216         ui->refcount = 0;
8217 done:
8218         sx_xunlock(&t4_uld_list_lock);
8219         return (rc);
8220 }
8221
8222 int
8223 t4_unregister_uld(struct uld_info *ui)
8224 {
8225         int rc = EINVAL;
8226         struct uld_info *u;
8227
8228         sx_xlock(&t4_uld_list_lock);
8229
8230         SLIST_FOREACH(u, &t4_uld_list, link) {
8231             if (u == ui) {
8232                     if (ui->refcount > 0) {
8233                             rc = EBUSY;
8234                             goto done;
8235                     }
8236
8237                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8238                     rc = 0;
8239                     goto done;
8240             }
8241         }
8242 done:
8243         sx_xunlock(&t4_uld_list_lock);
8244         return (rc);
8245 }
8246
8247 int
8248 t4_activate_uld(struct adapter *sc, int id)
8249 {
8250         int rc = EAGAIN;
8251         struct uld_info *ui;
8252
8253         ASSERT_SYNCHRONIZED_OP(sc);
8254
8255         sx_slock(&t4_uld_list_lock);
8256
8257         SLIST_FOREACH(ui, &t4_uld_list, link) {
8258                 if (ui->uld_id == id) {
8259                         rc = ui->activate(sc);
8260                         if (rc == 0)
8261                                 ui->refcount++;
8262                         goto done;
8263                 }
8264         }
8265 done:
8266         sx_sunlock(&t4_uld_list_lock);
8267
8268         return (rc);
8269 }
8270
8271 int
8272 t4_deactivate_uld(struct adapter *sc, int id)
8273 {
8274         int rc = EINVAL;
8275         struct uld_info *ui;
8276
8277         ASSERT_SYNCHRONIZED_OP(sc);
8278
8279         sx_slock(&t4_uld_list_lock);
8280
8281         SLIST_FOREACH(ui, &t4_uld_list, link) {
8282                 if (ui->uld_id == id) {
8283                         rc = ui->deactivate(sc);
8284                         if (rc == 0)
8285                                 ui->refcount--;
8286                         goto done;
8287                 }
8288         }
8289 done:
8290         sx_sunlock(&t4_uld_list_lock);
8291
8292         return (rc);
8293 }
8294 #endif
8295
8296 /*
8297  * Come up with reasonable defaults for some of the tunables, provided they're
8298  * not set by the user (in which case we'll use the values as is).
8299  */
8300 static void
8301 tweak_tunables(void)
8302 {
8303         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8304
8305         if (t4_ntxq10g < 1)
8306                 t4_ntxq10g = min(nc, NTXQ_10G);
8307
8308         if (t4_ntxq1g < 1)
8309                 t4_ntxq1g = min(nc, NTXQ_1G);
8310
8311         if (t4_nrxq10g < 1)
8312                 t4_nrxq10g = min(nc, NRXQ_10G);
8313
8314         if (t4_nrxq1g < 1)
8315                 t4_nrxq1g = min(nc, NRXQ_1G);
8316
8317 #ifdef TCP_OFFLOAD
8318         if (t4_nofldtxq10g < 1)
8319                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8320
8321         if (t4_nofldtxq1g < 1)
8322                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8323
8324         if (t4_nofldrxq10g < 1)
8325                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8326
8327         if (t4_nofldrxq1g < 1)
8328                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8329
8330         if (t4_toecaps_allowed == -1)
8331                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8332 #else
8333         if (t4_toecaps_allowed == -1)
8334                 t4_toecaps_allowed = 0;
8335 #endif
8336
8337 #ifdef DEV_NETMAP
8338         if (t4_nnmtxq10g < 1)
8339                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8340
8341         if (t4_nnmtxq1g < 1)
8342                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8343
8344         if (t4_nnmrxq10g < 1)
8345                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8346
8347         if (t4_nnmrxq1g < 1)
8348                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8349 #endif
8350
8351         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8352                 t4_tmr_idx_10g = TMR_IDX_10G;
8353
8354         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8355                 t4_pktc_idx_10g = PKTC_IDX_10G;
8356
8357         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8358                 t4_tmr_idx_1g = TMR_IDX_1G;
8359
8360         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8361                 t4_pktc_idx_1g = PKTC_IDX_1G;
8362
8363         if (t4_qsize_txq < 128)
8364                 t4_qsize_txq = 128;
8365
8366         if (t4_qsize_rxq < 128)
8367                 t4_qsize_rxq = 128;
8368         while (t4_qsize_rxq & 7)
8369                 t4_qsize_rxq++;
8370
8371         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8372 }
8373
8374 static struct sx mlu;   /* mod load unload */
8375 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8376
8377 static int
8378 mod_event(module_t mod, int cmd, void *arg)
8379 {
8380         int rc = 0;
8381         static int loaded = 0;
8382
8383         switch (cmd) {
8384         case MOD_LOAD:
8385                 sx_xlock(&mlu);
8386                 if (loaded++ == 0) {
8387                         t4_sge_modload();
8388                         sx_init(&t4_list_lock, "T4/T5 adapters");
8389                         SLIST_INIT(&t4_list);
8390 #ifdef TCP_OFFLOAD
8391                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8392                         SLIST_INIT(&t4_uld_list);
8393 #endif
8394                         t4_tracer_modload();
8395                         tweak_tunables();
8396                 }
8397                 sx_xunlock(&mlu);
8398                 break;
8399
8400         case MOD_UNLOAD:
8401                 sx_xlock(&mlu);
8402                 if (--loaded == 0) {
8403                         int tries;
8404
8405                         sx_slock(&t4_list_lock);
8406                         if (!SLIST_EMPTY(&t4_list)) {
8407                                 rc = EBUSY;
8408                                 sx_sunlock(&t4_list_lock);
8409                                 goto done_unload;
8410                         }
8411 #ifdef TCP_OFFLOAD
8412                         sx_slock(&t4_uld_list_lock);
8413                         if (!SLIST_EMPTY(&t4_uld_list)) {
8414                                 rc = EBUSY;
8415                                 sx_sunlock(&t4_uld_list_lock);
8416                                 sx_sunlock(&t4_list_lock);
8417                                 goto done_unload;
8418                         }
8419 #endif
8420                         tries = 0;
8421                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8422                                 uprintf("%ju clusters with custom free routine "
8423                                     "still is use.\n", t4_sge_extfree_refs());
8424                                 pause("t4unload", 2 * hz);
8425                         }
8426 #ifdef TCP_OFFLOAD
8427                         sx_sunlock(&t4_uld_list_lock);
8428 #endif
8429                         sx_sunlock(&t4_list_lock);
8430
8431                         if (t4_sge_extfree_refs() == 0) {
8432                                 t4_tracer_modunload();
8433 #ifdef TCP_OFFLOAD
8434                                 sx_destroy(&t4_uld_list_lock);
8435 #endif
8436                                 sx_destroy(&t4_list_lock);
8437                                 t4_sge_modunload();
8438                                 loaded = 0;
8439                         } else {
8440                                 rc = EBUSY;
8441                                 loaded++;       /* undo earlier decrement */
8442                         }
8443                 }
8444 done_unload:
8445                 sx_xunlock(&mlu);
8446                 break;
8447         }
8448
8449         return (rc);
8450 }
8451
8452 static devclass_t t4_devclass, t5_devclass;
8453 static devclass_t cxgbe_devclass, cxl_devclass;
8454
8455 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8456 MODULE_VERSION(t4nex, 1);
8457 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8458
8459 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8460 MODULE_VERSION(t5nex, 1);
8461 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8462
8463 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8464 MODULE_VERSION(cxgbe, 1);
8465
8466 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8467 MODULE_VERSION(cxl, 1);