sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static int
 574 t4_attach(device_t dev)
 575 {
 576         struct adapter *sc;
 577         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 578         struct intrs_and_queues iaq;
 579         struct sge *s;
 580 #ifdef TCP_OFFLOAD
 581         int ofld_rqidx, ofld_tqidx;
 582 #endif
 583 #ifdef DEV_NETMAP
 584         int nm_rqidx, nm_tqidx;
 585 #endif
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 590
 591         pci_enable_busmaster(dev);
 592         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 593                 uint32_t v;
 594
 595                 pci_set_max_read_req(dev, 4096);
 596                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 597                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 598                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 599
 600                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 601         }
 602
 603         sc->traceq = -1;
 604         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 605         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 606             device_get_nameunit(dev));
 607
 608         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 609             device_get_nameunit(dev));
 610         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 611         sx_xlock(&t4_list_lock);
 612         SLIST_INSERT_HEAD(&t4_list, sc, link);
 613         sx_xunlock(&t4_list_lock);
 614
 615         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 616         TAILQ_INIT(&sc->sfl);
 617         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 618
 619         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 620
 621         rc = map_bars_0_and_4(sc);
 622         if (rc != 0)
 623                 goto done; /* error message displayed already */
 624
 625         /*
 626          * This is the real PF# to which we're attaching.  Works from within PCI
 627          * passthrough environments too, where pci_get_function() could return a
 628          * different PF# depending on the passthrough configuration.  We need to
 629          * use the real PF# in all our communication with the firmware.
 630          */
 631         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 632         sc->mbox = sc->pf;
 633
 634         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 635         sc->an_handler = an_not_handled;
 636         for (i = 0; i < nitems(sc->cpl_handler); i++)
 637                 sc->cpl_handler[i] = cpl_not_handled;
 638         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 639                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 640         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 642         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 643         t4_init_sge_cpl_handlers(sc);
 644
 645         /* Prepare the adapter for operation */
 646         rc = -t4_prep_adapter(sc);
 647         if (rc != 0) {
 648                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 649                 goto done;
 650         }
 651
 652         /*
 653          * Do this really early, with the memory windows set up even before the
 654          * character device.  The userland tool's register i/o and mem read
 655          * will work even in "recovery mode".
 656          */
 657         setup_memwin(sc);
 658         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 659             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 660             device_get_nameunit(dev));
 661         if (sc->cdev == NULL)
 662                 device_printf(dev, "failed to create nexus char device.\n");
 663         else
 664                 sc->cdev->si_drv1 = sc;
 665
 666         /* Go no further if recovery mode has been requested. */
 667         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 668                 device_printf(dev, "recovery mode.\n");
 669                 goto done;
 670         }
 671
 672 #if defined(__i386__)
 673         if ((cpu_feature & CPUID_CX8) == 0) {
 674                 device_printf(dev, "64 bit atomics not available.\n");
 675                 rc = ENOTSUP;
 676                 goto done;
 677         }
 678 #endif
 679
 680         /* Prepare the firmware for operation */
 681         rc = prep_firmware(sc);
 682         if (rc != 0)
 683                 goto done; /* error message displayed already */
 684
 685         rc = get_params__post_init(sc);
 686         if (rc != 0)
 687                 goto done; /* error message displayed already */
 688
 689         rc = set_params__post_init(sc);
 690         if (rc != 0)
 691                 goto done; /* error message displayed already */
 692
 693         rc = map_bar_2(sc);
 694         if (rc != 0)
 695                 goto done; /* error message displayed already */
 696
 697         rc = t4_create_dma_tag(sc);
 698         if (rc != 0)
 699                 goto done; /* error message displayed already */
 700
 701         /*
 702          * First pass over all the ports - allocate VIs and initialize some
 703          * basic parameters like mac address, port type, etc.  We also figure
 704          * out whether a port is 10G or 1G and use that information when
 705          * calculating how many interrupts to attempt to allocate.
 706          */
 707         n10g = n1g = 0;
 708         for_each_port(sc, i) {
 709                 struct port_info *pi;
 710
 711                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 712                 sc->port[i] = pi;
 713
 714                 /* These must be set before t4_port_init */
 715                 pi->adapter = sc;
 716                 pi->port_id = i;
 717
 718                 /* Allocate the vi and initialize parameters like mac addr */
 719                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 720                 if (rc != 0) {
 721                         device_printf(dev, "unable to initialize port %d: %d\n",
 722                             i, rc);
 723                         free(pi, M_CXGBE);
 724                         sc->port[i] = NULL;
 725                         goto done;
 726                 }
 727
 728                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 729                 pi->link_cfg.requested_fc |= t4_pause_settings;
 730                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 731                 pi->link_cfg.fc |= t4_pause_settings;
 732
 733                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 734                 if (rc != 0) {
 735                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 736                         free(pi, M_CXGBE);
 737                         sc->port[i] = NULL;
 738                         goto done;
 739                 }
 740
 741                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 742                     device_get_nameunit(dev), i);
 743                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 744                 sc->chan_map[pi->tx_chan] = i;
 745
 746                 if (is_10G_port(pi) || is_40G_port(pi)) {
 747                         n10g++;
 748                         pi->tmr_idx = t4_tmr_idx_10g;
 749                         pi->pktc_idx = t4_pktc_idx_10g;
 750                 } else {
 751                         n1g++;
 752                         pi->tmr_idx = t4_tmr_idx_1g;
 753                         pi->pktc_idx = t4_pktc_idx_1g;
 754                 }
 755
 756                 pi->xact_addr_filt = -1;
 757                 pi->linkdnrc = -1;
 758
 759                 pi->qsize_rxq = t4_qsize_rxq;
 760                 pi->qsize_txq = t4_qsize_txq;
 761
 762                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 763                 if (pi->dev == NULL) {
 764                         device_printf(dev,
 765                             "failed to add device for port %d.\n", i);
 766                         rc = ENXIO;
 767                         goto done;
 768                 }
 769                 device_set_softc(pi->dev, pi);
 770         }
 771
 772         /*
 773          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 774          */
 775         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 776         if (rc != 0)
 777                 goto done; /* error message displayed already */
 778
 779         sc->intr_type = iaq.intr_type;
 780         sc->intr_count = iaq.nirq;
 781
 782         s = &sc->sge;
 783         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 784         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 785         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 786         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 787         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 788 #ifdef TCP_OFFLOAD
 789         if (is_offload(sc)) {
 790                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 791                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 792                 s->neq += s->nofldtxq + s->nofldrxq;
 793                 s->niq += s->nofldrxq;
 794
 795                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 796                     M_CXGBE, M_ZERO | M_WAITOK);
 797                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 798                     M_CXGBE, M_ZERO | M_WAITOK);
 799         }
 800 #endif
 801 #ifdef DEV_NETMAP
 802         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 803         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 804         s->neq += s->nnmtxq + s->nnmrxq;
 805         s->niq += s->nnmrxq;
 806
 807         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 808             M_CXGBE, M_ZERO | M_WAITOK);
 809         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 810             M_CXGBE, M_ZERO | M_WAITOK);
 811 #endif
 812
 813         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 814             M_ZERO | M_WAITOK);
 815         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 816             M_ZERO | M_WAITOK);
 817         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 818             M_ZERO | M_WAITOK);
 819         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 820             M_ZERO | M_WAITOK);
 821         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 822             M_ZERO | M_WAITOK);
 823
 824         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 825             M_ZERO | M_WAITOK);
 826
 827         t4_init_l2t(sc, M_WAITOK);
 828
 829         /*
 830          * Second pass over the ports.  This time we know the number of rx and
 831          * tx queues that each port should get.
 832          */
 833         rqidx = tqidx = 0;
 834 #ifdef TCP_OFFLOAD
 835         ofld_rqidx = ofld_tqidx = 0;
 836 #endif
 837 #ifdef DEV_NETMAP
 838         nm_rqidx = nm_tqidx = 0;
 839 #endif
 840         for_each_port(sc, i) {
 841                 struct port_info *pi = sc->port[i];
 842
 843                 if (pi == NULL)
 844                         continue;
 845
 846                 pi->first_rxq = rqidx;
 847                 pi->first_txq = tqidx;
 848                 if (is_10G_port(pi) || is_40G_port(pi)) {
 849                         pi->flags |= iaq.intr_flags_10g;
 850                         pi->nrxq = iaq.nrxq10g;
 851                         pi->ntxq = iaq.ntxq10g;
 852                 } else {
 853                         pi->flags |= iaq.intr_flags_1g;
 854                         pi->nrxq = iaq.nrxq1g;
 855                         pi->ntxq = iaq.ntxq1g;
 856                 }
 857
 858                 if (pi->ntxq > 1)
 859                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 860                 else
 861                         pi->rsrv_noflowq = 0;
 862
 863                 rqidx += pi->nrxq;
 864                 tqidx += pi->ntxq;
 865 #ifdef TCP_OFFLOAD
 866                 if (is_offload(sc)) {
 867                         pi->first_ofld_rxq = ofld_rqidx;
 868                         pi->first_ofld_txq = ofld_tqidx;
 869                         if (is_10G_port(pi) || is_40G_port(pi)) {
 870                                 pi->nofldrxq = iaq.nofldrxq10g;
 871                                 pi->nofldtxq = iaq.nofldtxq10g;
 872                         } else {
 873                                 pi->nofldrxq = iaq.nofldrxq1g;
 874                                 pi->nofldtxq = iaq.nofldtxq1g;
 875                         }
 876                         ofld_rqidx += pi->nofldrxq;
 877                         ofld_tqidx += pi->nofldtxq;
 878                 }
 879 #endif
 880 #ifdef DEV_NETMAP
 881                 pi->first_nm_rxq = nm_rqidx;
 882                 pi->first_nm_txq = nm_tqidx;
 883                 if (is_10G_port(pi) || is_40G_port(pi)) {
 884                         pi->nnmrxq = iaq.nnmrxq10g;
 885                         pi->nnmtxq = iaq.nnmtxq10g;
 886                 } else {
 887                         pi->nnmrxq = iaq.nnmrxq1g;
 888                         pi->nnmtxq = iaq.nnmtxq1g;
 889                 }
 890                 nm_rqidx += pi->nnmrxq;
 891                 nm_tqidx += pi->nnmtxq;
 892 #endif
 893         }
 894
 895         rc = setup_intr_handlers(sc);
 896         if (rc != 0) {
 897                 device_printf(dev,
 898                     "failed to setup interrupt handlers: %d\n", rc);
 899                 goto done;
 900         }
 901
 902         rc = bus_generic_attach(dev);
 903         if (rc != 0) {
 904                 device_printf(dev,
 905                     "failed to attach all child ports: %d\n", rc);
 906                 goto done;
 907         }
 908
 909         device_printf(dev,
 910             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 911             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 912             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 913             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 914             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 915
 916         t4_set_desc(sc);
 917
 918 done:
 919         if (rc != 0 && sc->cdev) {
 920                 /* cdev was created and so cxgbetool works; recover that way. */
 921                 device_printf(dev,
 922                     "error during attach, adapter is now in recovery mode.\n");
 923                 rc = 0;
 924         }
 925
 926         if (rc != 0)
 927                 t4_detach(dev);
 928         else
 929                 t4_sysctls(sc);
 930
 931         return (rc);
 932 }
 933
 934 /*
 935  * Idempotent
 936  */
 937 static int
 938 t4_detach(device_t dev)
 939 {
 940         struct adapter *sc;
 941         struct port_info *pi;
 942         int i, rc;
 943
 944         sc = device_get_softc(dev);
 945
 946         if (sc->flags & FULL_INIT_DONE)
 947                 t4_intr_disable(sc);
 948
 949         if (sc->cdev) {
 950                 destroy_dev(sc->cdev);
 951                 sc->cdev = NULL;
 952         }
 953
 954         rc = bus_generic_detach(dev);
 955         if (rc) {
 956                 device_printf(dev,
 957                     "failed to detach child devices: %d\n", rc);
 958                 return (rc);
 959         }
 960
 961         for (i = 0; i < sc->intr_count; i++)
 962                 t4_free_irq(sc, &sc->irq[i]);
 963
 964         for (i = 0; i < MAX_NPORTS; i++) {
 965                 pi = sc->port[i];
 966                 if (pi) {
 967                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 968                         if (pi->dev)
 969                                 device_delete_child(dev, pi->dev);
 970
 971                         mtx_destroy(&pi->pi_lock);
 972                         free(pi, M_CXGBE);
 973                 }
 974         }
 975
 976         if (sc->flags & FULL_INIT_DONE)
 977                 adapter_full_uninit(sc);
 978
 979         if (sc->flags & FW_OK)
 980                 t4_fw_bye(sc, sc->mbox);
 981
 982         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 983                 pci_release_msi(dev);
 984
 985         if (sc->regs_res)
 986                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 987                     sc->regs_res);
 988
 989         if (sc->udbs_res)
 990                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 991                     sc->udbs_res);
 992
 993         if (sc->msix_res)
 994                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 995                     sc->msix_res);
 996
 997         if (sc->l2t)
 998                 t4_free_l2t(sc->l2t);
 999
1000 #ifdef TCP_OFFLOAD
1001         free(sc->sge.ofld_rxq, M_CXGBE);
1002         free(sc->sge.ofld_txq, M_CXGBE);
1003 #endif
1004 #ifdef DEV_NETMAP
1005         free(sc->sge.nm_rxq, M_CXGBE);
1006         free(sc->sge.nm_txq, M_CXGBE);
1007 #endif
1008         free(sc->irq, M_CXGBE);
1009         free(sc->sge.rxq, M_CXGBE);
1010         free(sc->sge.txq, M_CXGBE);
1011         free(sc->sge.ctrlq, M_CXGBE);
1012         free(sc->sge.iqmap, M_CXGBE);
1013         free(sc->sge.eqmap, M_CXGBE);
1014         free(sc->tids.ftid_tab, M_CXGBE);
1015         t4_destroy_dma_tag(sc);
1016         if (mtx_initialized(&sc->sc_lock)) {
1017                 sx_xlock(&t4_list_lock);
1018                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1019                 sx_xunlock(&t4_list_lock);
1020                 mtx_destroy(&sc->sc_lock);
1021         }
1022
1023         if (mtx_initialized(&sc->tids.ftid_lock))
1024                 mtx_destroy(&sc->tids.ftid_lock);
1025         if (mtx_initialized(&sc->sfl_lock))
1026                 mtx_destroy(&sc->sfl_lock);
1027         if (mtx_initialized(&sc->ifp_lock))
1028                 mtx_destroy(&sc->ifp_lock);
1029         if (mtx_initialized(&sc->regwin_lock))
1030                 mtx_destroy(&sc->regwin_lock);
1031
1032         bzero(sc, sizeof(*sc));
1033
1034         return (0);
1035 }
1036
1037 static int
1038 cxgbe_probe(device_t dev)
1039 {
1040         char buf[128];
1041         struct port_info *pi = device_get_softc(dev);
1042
1043         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1044         device_set_desc_copy(dev, buf);
1045
1046         return (BUS_PROBE_DEFAULT);
1047 }
1048
1049 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1050     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1051     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1052 #define T4_CAP_ENABLE (T4_CAP)
1053
1054 static int
1055 cxgbe_attach(device_t dev)
1056 {
1057         struct port_info *pi = device_get_softc(dev);
1058         struct ifnet *ifp;
1059         char *s;
1060         int n, o;
1061
1062         /* Allocate an ifnet and set it up */
1063         ifp = if_alloc(IFT_ETHER);
1064         if (ifp == NULL) {
1065                 device_printf(dev, "Cannot allocate ifnet\n");
1066                 return (ENOMEM);
1067         }
1068         pi->ifp = ifp;
1069         ifp->if_softc = pi;
1070
1071         callout_init(&pi->tick, CALLOUT_MPSAFE);
1072
1073         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1074         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1075
1076         ifp->if_init = cxgbe_init;
1077         ifp->if_ioctl = cxgbe_ioctl;
1078         ifp->if_transmit = cxgbe_transmit;
1079         ifp->if_qflush = cxgbe_qflush;
1080
1081         ifp->if_capabilities = T4_CAP;
1082 #ifdef TCP_OFFLOAD
1083         if (is_offload(pi->adapter))
1084                 ifp->if_capabilities |= IFCAP_TOE;
1085 #endif
1086         ifp->if_capenable = T4_CAP_ENABLE;
1087         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1088             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1089
1090         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1091         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1092         ifp->if_hw_tsomaxsegsize = 65536;
1093
1094         /* Initialize ifmedia for this port */
1095         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1096             cxgbe_media_status);
1097         build_medialist(pi, &pi->media);
1098
1099         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1100             EVENTHANDLER_PRI_ANY);
1101
1102         ether_ifattach(ifp, pi->hw_addr);
1103
1104         n = 128;
1105         s = malloc(n, M_CXGBE, M_WAITOK);
1106         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1107         MPASS(n > o);
1108 #ifdef TCP_OFFLOAD
1109         if (is_offload(pi->adapter)) {
1110                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1111                     pi->nofldtxq, pi->nofldrxq);
1112                 MPASS(n > o);
1113         }
1114 #endif
1115 #ifdef DEV_NETMAP
1116         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1117             pi->nnmrxq);
1118         MPASS(n > o);
1119 #endif
1120         device_printf(dev, "%s\n", s);
1121         free(s, M_CXGBE);
1122
1123 #ifdef DEV_NETMAP
1124         /* nm_media handled here to keep implementation private to this file */
1125         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1126             cxgbe_media_status);
1127         build_medialist(pi, &pi->nm_media);
1128         create_netmap_ifnet(pi);        /* logs errors it something fails */
1129 #endif
1130         cxgbe_sysctls(pi);
1131
1132         return (0);
1133 }
1134
1135 static int
1136 cxgbe_detach(device_t dev)
1137 {
1138         struct port_info *pi = device_get_softc(dev);
1139         struct adapter *sc = pi->adapter;
1140         struct ifnet *ifp = pi->ifp;
1141
1142         /* Tell if_ioctl and if_init that the port is going away */
1143         ADAPTER_LOCK(sc);
1144         SET_DOOMED(pi);
1145         wakeup(&sc->flags);
1146         while (IS_BUSY(sc))
1147                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1148         SET_BUSY(sc);
1149 #ifdef INVARIANTS
1150         sc->last_op = "t4detach";
1151         sc->last_op_thr = curthread;
1152         sc->last_op_flags = 0;
1153 #endif
1154         ADAPTER_UNLOCK(sc);
1155
1156         if (pi->flags & HAS_TRACEQ) {
1157                 sc->traceq = -1;        /* cloner should not create ifnet */
1158                 t4_tracer_port_detach(sc);
1159         }
1160
1161         if (pi->vlan_c)
1162                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1163
1164         PORT_LOCK(pi);
1165         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1166         callout_stop(&pi->tick);
1167         PORT_UNLOCK(pi);
1168         callout_drain(&pi->tick);
1169
1170         /* Let detach proceed even if these fail. */
1171         cxgbe_uninit_synchronized(pi);
1172         port_full_uninit(pi);
1173
1174         ifmedia_removeall(&pi->media);
1175         ether_ifdetach(pi->ifp);
1176         if_free(pi->ifp);
1177
1178 #ifdef DEV_NETMAP
1179         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1180         destroy_netmap_ifnet(pi);
1181 #endif
1182
1183         ADAPTER_LOCK(sc);
1184         CLR_BUSY(sc);
1185         wakeup(&sc->flags);
1186         ADAPTER_UNLOCK(sc);
1187
1188         return (0);
1189 }
1190
1191 static void
1192 cxgbe_init(void *arg)
1193 {
1194         struct port_info *pi = arg;
1195         struct adapter *sc = pi->adapter;
1196
1197         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1198                 return;
1199         cxgbe_init_synchronized(pi);
1200         end_synchronized_op(sc, 0);
1201 }
1202
1203 static int
1204 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1205 {
1206         int rc = 0, mtu, flags, can_sleep;
1207         struct port_info *pi = ifp->if_softc;
1208         struct adapter *sc = pi->adapter;
1209         struct ifreq *ifr = (struct ifreq *)data;
1210         uint32_t mask;
1211
1212         switch (cmd) {
1213         case SIOCSIFMTU:
1214                 mtu = ifr->ifr_mtu;
1215                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1216                         return (EINVAL);
1217
1218                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1219                 if (rc)
1220                         return (rc);
1221                 ifp->if_mtu = mtu;
1222                 if (pi->flags & PORT_INIT_DONE) {
1223                         t4_update_fl_bufsize(ifp);
1224                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1225                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1226                 }
1227                 end_synchronized_op(sc, 0);
1228                 break;
1229
1230         case SIOCSIFFLAGS:
1231                 can_sleep = 0;
1232 redo_sifflags:
1233                 rc = begin_synchronized_op(sc, pi,
1234                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1235                 if (rc)
1236                         return (rc);
1237
1238                 if (ifp->if_flags & IFF_UP) {
1239                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1240                                 flags = pi->if_flags;
1241                                 if ((ifp->if_flags ^ flags) &
1242                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1243                                         if (can_sleep == 1) {
1244                                                 end_synchronized_op(sc, 0);
1245                                                 can_sleep = 0;
1246                                                 goto redo_sifflags;
1247                                         }
1248                                         rc = update_mac_settings(ifp,
1249                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1250                                 }
1251                         } else {
1252                                 if (can_sleep == 0) {
1253                                         end_synchronized_op(sc, LOCK_HELD);
1254                                         can_sleep = 1;
1255                                         goto redo_sifflags;
1256                                 }
1257                                 rc = cxgbe_init_synchronized(pi);
1258                         }
1259                         pi->if_flags = ifp->if_flags;
1260                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1261                         if (can_sleep == 0) {
1262                                 end_synchronized_op(sc, LOCK_HELD);
1263                                 can_sleep = 1;
1264                                 goto redo_sifflags;
1265                         }
1266                         rc = cxgbe_uninit_synchronized(pi);
1267                 }
1268                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1269                 break;
1270
1271         case SIOCADDMULTI:
1272         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1273                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1274                 if (rc)
1275                         return (rc);
1276                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1277                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1278                 end_synchronized_op(sc, LOCK_HELD);
1279                 break;
1280
1281         case SIOCSIFCAP:
1282                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1283                 if (rc)
1284                         return (rc);
1285
1286                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1287                 if (mask & IFCAP_TXCSUM) {
1288                         ifp->if_capenable ^= IFCAP_TXCSUM;
1289                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1290
1291                         if (IFCAP_TSO4 & ifp->if_capenable &&
1292                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1293                                 ifp->if_capenable &= ~IFCAP_TSO4;
1294                                 if_printf(ifp,
1295                                     "tso4 disabled due to -txcsum.\n");
1296                         }
1297                 }
1298                 if (mask & IFCAP_TXCSUM_IPV6) {
1299                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1300                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1301
1302                         if (IFCAP_TSO6 & ifp->if_capenable &&
1303                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1304                                 ifp->if_capenable &= ~IFCAP_TSO6;
1305                                 if_printf(ifp,
1306                                     "tso6 disabled due to -txcsum6.\n");
1307                         }
1308                 }
1309                 if (mask & IFCAP_RXCSUM)
1310                         ifp->if_capenable ^= IFCAP_RXCSUM;
1311                 if (mask & IFCAP_RXCSUM_IPV6)
1312                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1313
1314                 /*
1315                  * Note that we leave CSUM_TSO alone (it is always set).  The
1316                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1317                  * sending a TSO request our way, so it's sufficient to toggle
1318                  * IFCAP_TSOx only.
1319                  */
1320                 if (mask & IFCAP_TSO4) {
1321                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1322                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1323                                 if_printf(ifp, "enable txcsum first.\n");
1324                                 rc = EAGAIN;
1325                                 goto fail;
1326                         }
1327                         ifp->if_capenable ^= IFCAP_TSO4;
1328                 }
1329                 if (mask & IFCAP_TSO6) {
1330                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1331                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1332                                 if_printf(ifp, "enable txcsum6 first.\n");
1333                                 rc = EAGAIN;
1334                                 goto fail;
1335                         }
1336                         ifp->if_capenable ^= IFCAP_TSO6;
1337                 }
1338                 if (mask & IFCAP_LRO) {
1339 #if defined(INET) || defined(INET6)
1340                         int i;
1341                         struct sge_rxq *rxq;
1342
1343                         ifp->if_capenable ^= IFCAP_LRO;
1344                         for_each_rxq(pi, i, rxq) {
1345                                 if (ifp->if_capenable & IFCAP_LRO)
1346                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1347                                 else
1348                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1349                         }
1350 #endif
1351                 }
1352 #ifdef TCP_OFFLOAD
1353                 if (mask & IFCAP_TOE) {
1354                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1355
1356                         rc = toe_capability(pi, enable);
1357                         if (rc != 0)
1358                                 goto fail;
1359
1360                         ifp->if_capenable ^= mask;
1361                 }
1362 #endif
1363                 if (mask & IFCAP_VLAN_HWTAGGING) {
1364                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1365                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1366                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1367                 }
1368                 if (mask & IFCAP_VLAN_MTU) {
1369                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1370
1371                         /* Need to find out how to disable auto-mtu-inflation */
1372                 }
1373                 if (mask & IFCAP_VLAN_HWTSO)
1374                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1375                 if (mask & IFCAP_VLAN_HWCSUM)
1376                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1377
1378 #ifdef VLAN_CAPABILITIES
1379                 VLAN_CAPABILITIES(ifp);
1380 #endif
1381 fail:
1382                 end_synchronized_op(sc, 0);
1383                 break;
1384
1385         case SIOCSIFMEDIA:
1386         case SIOCGIFMEDIA:
1387                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1388                 break;
1389
1390         case SIOCGI2C: {
1391                 struct ifi2creq i2c;
1392
1393                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1394                 if (rc != 0)
1395                         break;
1396                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1397                         rc = EPERM;
1398                         break;
1399                 }
1400                 if (i2c.len > sizeof(i2c.data)) {
1401                         rc = EINVAL;
1402                         break;
1403                 }
1404                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1405                 if (rc)
1406                         return (rc);
1407                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1408                     i2c.offset, i2c.len, &i2c.data[0]);
1409                 end_synchronized_op(sc, 0);
1410                 if (rc == 0)
1411                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1412                 break;
1413         }
1414
1415         default:
1416                 rc = ether_ioctl(ifp, cmd, data);
1417         }
1418
1419         return (rc);
1420 }
1421
1422 static int
1423 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1424 {
1425         struct port_info *pi = ifp->if_softc;
1426         struct adapter *sc = pi->adapter;
1427         struct sge_txq *txq;
1428         void *items[1];
1429         int rc;
1430
1431         M_ASSERTPKTHDR(m);
1432         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1433
1434         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1435                 m_freem(m);
1436                 return (ENETDOWN);
1437         }
1438
1439         rc = parse_pkt(&m);
1440         if (__predict_false(rc != 0)) {
1441                 MPASS(m == NULL);                       /* was freed already */
1442                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1443                 return (rc);
1444         }
1445
1446         /* Select a txq. */
1447         txq = &sc->sge.txq[pi->first_txq];
1448         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1449                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1450                     pi->rsrv_noflowq);
1451
1452         items[0] = m;
1453         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1454         if (__predict_false(rc != 0))
1455                 m_freem(m);
1456
1457         return (rc);
1458 }
1459
1460 static void
1461 cxgbe_qflush(struct ifnet *ifp)
1462 {
1463         struct port_info *pi = ifp->if_softc;
1464         struct sge_txq *txq;
1465         int i;
1466
1467         /* queues do not exist if !PORT_INIT_DONE. */
1468         if (pi->flags & PORT_INIT_DONE) {
1469                 for_each_txq(pi, i, txq) {
1470                         TXQ_LOCK(txq);
1471                         txq->eq.flags &= ~EQ_ENABLED;
1472                         TXQ_UNLOCK(txq);
1473                         while (!mp_ring_is_idle(txq->r)) {
1474                                 mp_ring_check_drainage(txq->r, 0);
1475                                 pause("qflush", 1);
1476                         }
1477                 }
1478         }
1479         if_qflush(ifp);
1480 }
1481
1482 static int
1483 cxgbe_media_change(struct ifnet *ifp)
1484 {
1485         struct port_info *pi = ifp->if_softc;
1486
1487         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1488
1489         return (EOPNOTSUPP);
1490 }
1491
1492 static void
1493 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1494 {
1495         struct port_info *pi = ifp->if_softc;
1496         struct ifmedia *media = NULL;
1497         struct ifmedia_entry *cur;
1498         int speed = pi->link_cfg.speed;
1499 #ifdef INVARIANTS
1500         int data = (pi->port_type << 8) | pi->mod_type;
1501 #endif
1502
1503         if (ifp == pi->ifp)
1504                 media = &pi->media;
1505 #ifdef DEV_NETMAP
1506         else if (ifp == pi->nm_ifp)
1507                 media = &pi->nm_media;
1508 #endif
1509         MPASS(media != NULL);
1510
1511         cur = media->ifm_cur;
1512         MPASS(cur->ifm_data == data);
1513
1514         ifmr->ifm_status = IFM_AVALID;
1515         if (!pi->link_cfg.link_ok)
1516                 return;
1517
1518         ifmr->ifm_status |= IFM_ACTIVE;
1519
1520         /* active and current will differ iff current media is autoselect. */
1521         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1522                 return;
1523
1524         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1525         if (speed == SPEED_10000)
1526                 ifmr->ifm_active |= IFM_10G_T;
1527         else if (speed == SPEED_1000)
1528                 ifmr->ifm_active |= IFM_1000_T;
1529         else if (speed == SPEED_100)
1530                 ifmr->ifm_active |= IFM_100_TX;
1531         else if (speed == SPEED_10)
1532                 ifmr->ifm_active |= IFM_10_T;
1533         else
1534                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1535                             speed));
1536 }
1537
1538 void
1539 t4_fatal_err(struct adapter *sc)
1540 {
1541         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1542         t4_intr_disable(sc);
1543         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1544             device_get_nameunit(sc->dev));
1545 }
1546
1547 static int
1548 map_bars_0_and_4(struct adapter *sc)
1549 {
1550         sc->regs_rid = PCIR_BAR(0);
1551         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1552             &sc->regs_rid, RF_ACTIVE);
1553         if (sc->regs_res == NULL) {
1554                 device_printf(sc->dev, "cannot map registers.\n");
1555                 return (ENXIO);
1556         }
1557         sc->bt = rman_get_bustag(sc->regs_res);
1558         sc->bh = rman_get_bushandle(sc->regs_res);
1559         sc->mmio_len = rman_get_size(sc->regs_res);
1560         setbit(&sc->doorbells, DOORBELL_KDB);
1561
1562         sc->msix_rid = PCIR_BAR(4);
1563         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1564             &sc->msix_rid, RF_ACTIVE);
1565         if (sc->msix_res == NULL) {
1566                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1567                 return (ENXIO);
1568         }
1569
1570         return (0);
1571 }
1572
1573 static int
1574 map_bar_2(struct adapter *sc)
1575 {
1576
1577         /*
1578          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1579          * to map it if RDMA is disabled.
1580          */
1581         if (is_t4(sc) && sc->rdmacaps == 0)
1582                 return (0);
1583
1584         sc->udbs_rid = PCIR_BAR(2);
1585         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1586             &sc->udbs_rid, RF_ACTIVE);
1587         if (sc->udbs_res == NULL) {
1588                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1589                 return (ENXIO);
1590         }
1591         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1592
1593         if (is_t5(sc)) {
1594                 setbit(&sc->doorbells, DOORBELL_UDB);
1595 #if defined(__i386__) || defined(__amd64__)
1596                 if (t5_write_combine) {
1597                         int rc;
1598
1599                         /*
1600                          * Enable write combining on BAR2.  This is the
1601                          * userspace doorbell BAR and is split into 128B
1602                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1603                          * with an egress queue.  The first 64B has the doorbell
1604                          * and the second 64B can be used to submit a tx work
1605                          * request with an implicit doorbell.
1606                          */
1607
1608                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1609                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1610                         if (rc == 0) {
1611                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1612                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1613                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1614                         } else {
1615                                 device_printf(sc->dev,
1616                                     "couldn't enable write combining: %d\n",
1617                                     rc);
1618                         }
1619
1620                         t4_write_reg(sc, A_SGE_STAT_CFG,
1621                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1622                 }
1623 #endif
1624         }
1625
1626         return (0);
1627 }
1628
1629 static const struct memwin t4_memwin[] = {
1630         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1631         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1632         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1633 };
1634
1635 static const struct memwin t5_memwin[] = {
1636         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1637         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1638         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1639 };
1640
1641 static void
1642 setup_memwin(struct adapter *sc)
1643 {
1644         const struct memwin *mw;
1645         int i, n;
1646         uint32_t bar0;
1647
1648         if (is_t4(sc)) {
1649                 /*
1650                  * Read low 32b of bar0 indirectly via the hardware backdoor
1651                  * mechanism.  Works from within PCI passthrough environments
1652                  * too, where rman_get_start() can return a different value.  We
1653                  * need to program the T4 memory window decoders with the actual
1654                  * addresses that will be coming across the PCIe link.
1655                  */
1656                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1657                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1658
1659                 mw = &t4_memwin[0];
1660                 n = nitems(t4_memwin);
1661         } else {
1662                 /* T5 uses the relative offset inside the PCIe BAR */
1663                 bar0 = 0;
1664
1665                 mw = &t5_memwin[0];
1666                 n = nitems(t5_memwin);
1667         }
1668
1669         for (i = 0; i < n; i++, mw++) {
1670                 t4_write_reg(sc,
1671                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1672                     (mw->base + bar0) | V_BIR(0) |
1673                     V_WINDOW(ilog2(mw->aperture) - 10));
1674         }
1675
1676         /* flush */
1677         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1678 }
1679
1680 /*
1681  * Verify that the memory range specified by the addr/len pair is valid and lies
1682  * entirely within a single region (EDCx or MCx).
1683  */
1684 static int
1685 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1686 {
1687         uint32_t em, addr_len, maddr, mlen;
1688
1689         /* Memory can only be accessed in naturally aligned 4 byte units */
1690         if (addr & 3 || len & 3 || len == 0)
1691                 return (EINVAL);
1692
1693         /* Enabled memories */
1694         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1695         if (em & F_EDRAM0_ENABLE) {
1696                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1697                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1698                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1699                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1700                     addr + len <= maddr + mlen)
1701                         return (0);
1702         }
1703         if (em & F_EDRAM1_ENABLE) {
1704                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1705                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1706                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1707                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1708                     addr + len <= maddr + mlen)
1709                         return (0);
1710         }
1711         if (em & F_EXT_MEM_ENABLE) {
1712                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1713                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1714                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1715                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1716                     addr + len <= maddr + mlen)
1717                         return (0);
1718         }
1719         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1720                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1721                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1722                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1723                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1724                     addr + len <= maddr + mlen)
1725                         return (0);
1726         }
1727
1728         return (EFAULT);
1729 }
1730
1731 static int
1732 fwmtype_to_hwmtype(int mtype)
1733 {
1734
1735         switch (mtype) {
1736         case FW_MEMTYPE_EDC0:
1737                 return (MEM_EDC0);
1738         case FW_MEMTYPE_EDC1:
1739                 return (MEM_EDC1);
1740         case FW_MEMTYPE_EXTMEM:
1741                 return (MEM_MC0);
1742         case FW_MEMTYPE_EXTMEM1:
1743                 return (MEM_MC1);
1744         default:
1745                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1746         }
1747 }
1748
1749 /*
1750  * Verify that the memory range specified by the memtype/offset/len pair is
1751  * valid and lies entirely within the memtype specified.  The global address of
1752  * the start of the range is returned in addr.
1753  */
1754 static int
1755 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1756     uint32_t *addr)
1757 {
1758         uint32_t em, addr_len, maddr, mlen;
1759
1760         /* Memory can only be accessed in naturally aligned 4 byte units */
1761         if (off & 3 || len & 3 || len == 0)
1762                 return (EINVAL);
1763
1764         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1765         switch (fwmtype_to_hwmtype(mtype)) {
1766         case MEM_EDC0:
1767                 if (!(em & F_EDRAM0_ENABLE))
1768                         return (EINVAL);
1769                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1770                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1771                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1772                 break;
1773         case MEM_EDC1:
1774                 if (!(em & F_EDRAM1_ENABLE))
1775                         return (EINVAL);
1776                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1777                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1778                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1779                 break;
1780         case MEM_MC:
1781                 if (!(em & F_EXT_MEM_ENABLE))
1782                         return (EINVAL);
1783                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1784                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1785                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1786                 break;
1787         case MEM_MC1:
1788                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1789                         return (EINVAL);
1790                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1791                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1792                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1793                 break;
1794         default:
1795                 return (EINVAL);
1796         }
1797
1798         if (mlen > 0 && off < mlen && off + len <= mlen) {
1799                 *addr = maddr + off;    /* global address */
1800                 return (0);
1801         }
1802
1803         return (EFAULT);
1804 }
1805
1806 static void
1807 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1808 {
1809         const struct memwin *mw;
1810
1811         if (is_t4(sc)) {
1812                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1813                     ("%s: incorrect memwin# (%d)", __func__, win));
1814                 mw = &t4_memwin[win];
1815         } else {
1816                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1817                     ("%s: incorrect memwin# (%d)", __func__, win));
1818                 mw = &t5_memwin[win];
1819         }
1820
1821         if (base != NULL)
1822                 *base = mw->base;
1823         if (aperture != NULL)
1824                 *aperture = mw->aperture;
1825 }
1826
1827 /*
1828  * Positions the memory window such that it can be used to access the specified
1829  * address in the chip's address space.  The return value is the offset of addr
1830  * from the start of the window.
1831  */
1832 static uint32_t
1833 position_memwin(struct adapter *sc, int n, uint32_t addr)
1834 {
1835         uint32_t start, pf;
1836         uint32_t reg;
1837
1838         KASSERT(n >= 0 && n <= 3,
1839             ("%s: invalid window %d.", __func__, n));
1840         KASSERT((addr & 3) == 0,
1841             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1842
1843         if (is_t4(sc)) {
1844                 pf = 0;
1845                 start = addr & ~0xf;    /* start must be 16B aligned */
1846         } else {
1847                 pf = V_PFNUM(sc->pf);
1848                 start = addr & ~0x7f;   /* start must be 128B aligned */
1849         }
1850         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1851
1852         t4_write_reg(sc, reg, start | pf);
1853         t4_read_reg(sc, reg);
1854
1855         return (addr - start);
1856 }
1857
1858 static int
1859 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1860     struct intrs_and_queues *iaq)
1861 {
1862         int rc, itype, navail, nrxq10g, nrxq1g, n;
1863         int nofldrxq10g = 0, nofldrxq1g = 0;
1864         int nnmrxq10g = 0, nnmrxq1g = 0;
1865
1866         bzero(iaq, sizeof(*iaq));
1867
1868         iaq->ntxq10g = t4_ntxq10g;
1869         iaq->ntxq1g = t4_ntxq1g;
1870         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1871         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1872         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1873 #ifdef TCP_OFFLOAD
1874         if (is_offload(sc)) {
1875                 iaq->nofldtxq10g = t4_nofldtxq10g;
1876                 iaq->nofldtxq1g = t4_nofldtxq1g;
1877                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1878                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1879         }
1880 #endif
1881 #ifdef DEV_NETMAP
1882         iaq->nnmtxq10g = t4_nnmtxq10g;
1883         iaq->nnmtxq1g = t4_nnmtxq1g;
1884         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1885         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1886 #endif
1887
1888         for (itype = INTR_MSIX; itype; itype >>= 1) {
1889
1890                 if ((itype & t4_intr_types) == 0)
1891                         continue;       /* not allowed */
1892
1893                 if (itype == INTR_MSIX)
1894                         navail = pci_msix_count(sc->dev);
1895                 else if (itype == INTR_MSI)
1896                         navail = pci_msi_count(sc->dev);
1897                 else
1898                         navail = 1;
1899 restart:
1900                 if (navail == 0)
1901                         continue;
1902
1903                 iaq->intr_type = itype;
1904                 iaq->intr_flags_10g = 0;
1905                 iaq->intr_flags_1g = 0;
1906
1907                 /*
1908                  * Best option: an interrupt vector for errors, one for the
1909                  * firmware event queue, and one for every rxq (NIC, TOE, and
1910                  * netmap).
1911                  */
1912                 iaq->nirq = T4_EXTRA_INTR;
1913                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1914                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1915                 if (iaq->nirq <= navail &&
1916                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1917                         iaq->intr_flags_10g = INTR_ALL;
1918                         iaq->intr_flags_1g = INTR_ALL;
1919                         goto allocate;
1920                 }
1921
1922                 /*
1923                  * Second best option: a vector for errors, one for the firmware
1924                  * event queue, and vectors for either all the NIC rx queues or
1925                  * all the TOE rx queues.  The queues that don't get vectors
1926                  * will forward their interrupts to those that do.
1927                  *
1928                  * Note: netmap rx queues cannot be created early and so they
1929                  * can't be setup to receive forwarded interrupts for others.
1930                  */
1931                 iaq->nirq = T4_EXTRA_INTR;
1932                 if (nrxq10g >= nofldrxq10g) {
1933                         iaq->intr_flags_10g = INTR_RXQ;
1934                         iaq->nirq += n10g * nrxq10g;
1935 #ifdef DEV_NETMAP
1936                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1937 #endif
1938                 } else {
1939                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1940                         iaq->nirq += n10g * nofldrxq10g;
1941 #ifdef DEV_NETMAP
1942                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1943 #endif
1944                 }
1945                 if (nrxq1g >= nofldrxq1g) {
1946                         iaq->intr_flags_1g = INTR_RXQ;
1947                         iaq->nirq += n1g * nrxq1g;
1948 #ifdef DEV_NETMAP
1949                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1950 #endif
1951                 } else {
1952                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1953                         iaq->nirq += n1g * nofldrxq1g;
1954 #ifdef DEV_NETMAP
1955                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1956 #endif
1957                 }
1958                 if (iaq->nirq <= navail &&
1959                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1960                         goto allocate;
1961
1962                 /*
1963                  * Next best option: an interrupt vector for errors, one for the
1964                  * firmware event queue, and at least one per port.  At this
1965                  * point we know we'll have to downsize nrxq and/or nofldrxq
1966                  * and/or nnmrxq to fit what's available to us.
1967                  */
1968                 iaq->nirq = T4_EXTRA_INTR;
1969                 iaq->nirq += n10g + n1g;
1970                 if (iaq->nirq <= navail) {
1971                         int leftover = navail - iaq->nirq;
1972
1973                         if (n10g > 0) {
1974                                 int target = max(nrxq10g, nofldrxq10g);
1975
1976                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1977                                     INTR_RXQ : INTR_OFLD_RXQ;
1978
1979                                 n = 1;
1980                                 while (n < target && leftover >= n10g) {
1981                                         leftover -= n10g;
1982                                         iaq->nirq += n10g;
1983                                         n++;
1984                                 }
1985                                 iaq->nrxq10g = min(n, nrxq10g);
1986 #ifdef TCP_OFFLOAD
1987                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1988 #endif
1989 #ifdef DEV_NETMAP
1990                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1991 #endif
1992                         }
1993
1994                         if (n1g > 0) {
1995                                 int target = max(nrxq1g, nofldrxq1g);
1996
1997                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1998                                     INTR_RXQ : INTR_OFLD_RXQ;
1999
2000                                 n = 1;
2001                                 while (n < target && leftover >= n1g) {
2002                                         leftover -= n1g;
2003                                         iaq->nirq += n1g;
2004                                         n++;
2005                                 }
2006                                 iaq->nrxq1g = min(n, nrxq1g);
2007 #ifdef TCP_OFFLOAD
2008                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2009 #endif
2010 #ifdef DEV_NETMAP
2011                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2012 #endif
2013                         }
2014
2015                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2016                                 goto allocate;
2017                 }
2018
2019                 /*
2020                  * Least desirable option: one interrupt vector for everything.
2021                  */
2022                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2023                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2024 #ifdef TCP_OFFLOAD
2025                 if (is_offload(sc))
2026                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2027 #endif
2028 #ifdef DEV_NETMAP
2029                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2030 #endif
2031
2032 allocate:
2033                 navail = iaq->nirq;
2034                 rc = 0;
2035                 if (itype == INTR_MSIX)
2036                         rc = pci_alloc_msix(sc->dev, &navail);
2037                 else if (itype == INTR_MSI)
2038                         rc = pci_alloc_msi(sc->dev, &navail);
2039
2040                 if (rc == 0) {
2041                         if (navail == iaq->nirq)
2042                                 return (0);
2043
2044                         /*
2045                          * Didn't get the number requested.  Use whatever number
2046                          * the kernel is willing to allocate (it's in navail).
2047                          */
2048                         device_printf(sc->dev, "fewer vectors than requested, "
2049                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2050                             itype, iaq->nirq, navail);
2051                         pci_release_msi(sc->dev);
2052                         goto restart;
2053                 }
2054
2055                 device_printf(sc->dev,
2056                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2057                     itype, rc, iaq->nirq, navail);
2058         }
2059
2060         device_printf(sc->dev,
2061             "failed to find a usable interrupt type.  "
2062             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2063             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2064
2065         return (ENXIO);
2066 }
2067
2068 #define FW_VERSION(chip) ( \
2069     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2070     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2071     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2072     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2073 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2074
2075 struct fw_info {
2076         uint8_t chip;
2077         char *kld_name;
2078         char *fw_mod_name;
2079         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2080 } fw_info[] = {
2081         {
2082                 .chip = CHELSIO_T4,
2083                 .kld_name = "t4fw_cfg",
2084                 .fw_mod_name = "t4fw",
2085                 .fw_hdr = {
2086                         .chip = FW_HDR_CHIP_T4,
2087                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2088                         .intfver_nic = FW_INTFVER(T4, NIC),
2089                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2090                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2091                         .intfver_ri = FW_INTFVER(T4, RI),
2092                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2093                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2094                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2095                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2096                 },
2097         }, {
2098                 .chip = CHELSIO_T5,
2099                 .kld_name = "t5fw_cfg",
2100                 .fw_mod_name = "t5fw",
2101                 .fw_hdr = {
2102                         .chip = FW_HDR_CHIP_T5,
2103                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2104                         .intfver_nic = FW_INTFVER(T5, NIC),
2105                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2106                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2107                         .intfver_ri = FW_INTFVER(T5, RI),
2108                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2109                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2110                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2111                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2112                 },
2113         }
2114 };
2115
2116 static struct fw_info *
2117 find_fw_info(int chip)
2118 {
2119         int i;
2120
2121         for (i = 0; i < nitems(fw_info); i++) {
2122                 if (fw_info[i].chip == chip)
2123                         return (&fw_info[i]);
2124         }
2125         return (NULL);
2126 }
2127
2128 /*
2129  * Is the given firmware API compatible with the one the driver was compiled
2130  * with?
2131  */
2132 static int
2133 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2134 {
2135
2136         /* short circuit if it's the exact same firmware version */
2137         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2138                 return (1);
2139
2140         /*
2141          * XXX: Is this too conservative?  Perhaps I should limit this to the
2142          * features that are supported in the driver.
2143          */
2144 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2145         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2146             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2147             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2148                 return (1);
2149 #undef SAME_INTF
2150
2151         return (0);
2152 }
2153
2154 /*
2155  * The firmware in the KLD is usable, but should it be installed?  This routine
2156  * explains itself in detail if it indicates the KLD firmware should be
2157  * installed.
2158  */
2159 static int
2160 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2161 {
2162         const char *reason;
2163
2164         if (!card_fw_usable) {
2165                 reason = "incompatible or unusable";
2166                 goto install;
2167         }
2168
2169         if (k > c) {
2170                 reason = "older than the version bundled with this driver";
2171                 goto install;
2172         }
2173
2174         if (t4_fw_install == 2 && k != c) {
2175                 reason = "different than the version bundled with this driver";
2176                 goto install;
2177         }
2178
2179         return (0);
2180
2181 install:
2182         if (t4_fw_install == 0) {
2183                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2184                     "but the driver is prohibited from installing a different "
2185                     "firmware on the card.\n",
2186                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2187                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2188
2189                 return (0);
2190         }
2191
2192         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2193             "installing firmware %u.%u.%u.%u on card.\n",
2194             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2195             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2196             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2197             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2198
2199         return (1);
2200 }
2201 /*
2202  * Establish contact with the firmware and determine if we are the master driver
2203  * or not, and whether we are responsible for chip initialization.
2204  */
2205 static int
2206 prep_firmware(struct adapter *sc)
2207 {
2208         const struct firmware *fw = NULL, *default_cfg;
2209         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2210         enum dev_state state;
2211         struct fw_info *fw_info;
2212         struct fw_hdr *card_fw;         /* fw on the card */
2213         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2214         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2215                                            against */
2216
2217         /* Contact firmware. */
2218         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2219         if (rc < 0 || state == DEV_STATE_ERR) {
2220                 rc = -rc;
2221                 device_printf(sc->dev,
2222                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2223                 return (rc);
2224         }
2225         pf = rc;
2226         if (pf == sc->mbox)
2227                 sc->flags |= MASTER_PF;
2228         else if (state == DEV_STATE_UNINIT) {
2229                 /*
2230                  * We didn't get to be the master so we definitely won't be
2231                  * configuring the chip.  It's a bug if someone else hasn't
2232                  * configured it already.
2233                  */
2234                 device_printf(sc->dev, "couldn't be master(%d), "
2235                     "device not already initialized either(%d).\n", rc, state);
2236                 return (EDOOFUS);
2237         }
2238
2239         /* This is the firmware whose headers the driver was compiled against */
2240         fw_info = find_fw_info(chip_id(sc));
2241         if (fw_info == NULL) {
2242                 device_printf(sc->dev,
2243                     "unable to look up firmware information for chip %d.\n",
2244                     chip_id(sc));
2245                 return (EINVAL);
2246         }
2247         drv_fw = &fw_info->fw_hdr;
2248
2249         /*
2250          * The firmware KLD contains many modules.  The KLD name is also the
2251          * name of the module that contains the default config file.
2252          */
2253         default_cfg = firmware_get(fw_info->kld_name);
2254
2255         /* Read the header of the firmware on the card */
2256         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2257         rc = -t4_read_flash(sc, FLASH_FW_START,
2258             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2259         if (rc == 0)
2260                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2261         else {
2262                 device_printf(sc->dev,
2263                     "Unable to read card's firmware header: %d\n", rc);
2264                 card_fw_usable = 0;
2265         }
2266
2267         /* This is the firmware in the KLD */
2268         fw = firmware_get(fw_info->fw_mod_name);
2269         if (fw != NULL) {
2270                 kld_fw = (const void *)fw->data;
2271                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2272         } else {
2273                 kld_fw = NULL;
2274                 kld_fw_usable = 0;
2275         }
2276
2277         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2278             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2279                 /*
2280                  * Common case: the firmware on the card is an exact match and
2281                  * the KLD is an exact match too, or the KLD is
2282                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2283                  * here -- use cxgbetool loadfw if you want to reinstall the
2284                  * same firmware as the one on the card.
2285                  */
2286         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2287             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2288             be32toh(card_fw->fw_ver))) {
2289
2290                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2291                 if (rc != 0) {
2292                         device_printf(sc->dev,
2293                             "failed to install firmware: %d\n", rc);
2294                         goto done;
2295                 }
2296
2297                 /* Installed successfully, update the cached header too. */
2298                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2299                 card_fw_usable = 1;
2300                 need_fw_reset = 0;      /* already reset as part of load_fw */
2301         }
2302
2303         if (!card_fw_usable) {
2304                 uint32_t d, c, k;
2305
2306                 d = ntohl(drv_fw->fw_ver);
2307                 c = ntohl(card_fw->fw_ver);
2308                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2309
2310                 device_printf(sc->dev, "Cannot find a usable firmware: "
2311                     "fw_install %d, chip state %d, "
2312                     "driver compiled with %d.%d.%d.%d, "
2313                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2314                     t4_fw_install, state,
2315                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2316                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2317                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2318                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2319                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2320                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2321                 rc = EINVAL;
2322                 goto done;
2323         }
2324
2325         /* We're using whatever's on the card and it's known to be good. */
2326         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2327         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2328             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2329             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2330             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2331             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2332         t4_get_tp_version(sc, &sc->params.tp_vers);
2333
2334         /* Reset device */
2335         if (need_fw_reset &&
2336             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2337                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2338                 if (rc != ETIMEDOUT && rc != EIO)
2339                         t4_fw_bye(sc, sc->mbox);
2340                 goto done;
2341         }
2342         sc->flags |= FW_OK;
2343
2344         rc = get_params__pre_init(sc);
2345         if (rc != 0)
2346                 goto done; /* error message displayed already */
2347
2348         /* Partition adapter resources as specified in the config file. */
2349         if (state == DEV_STATE_UNINIT) {
2350
2351                 KASSERT(sc->flags & MASTER_PF,
2352                     ("%s: trying to change chip settings when not master.",
2353                     __func__));
2354
2355                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2356                 if (rc != 0)
2357                         goto done;      /* error message displayed already */
2358
2359                 t4_tweak_chip_settings(sc);
2360
2361                 /* get basic stuff going */
2362                 rc = -t4_fw_initialize(sc, sc->mbox);
2363                 if (rc != 0) {
2364                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2365                         goto done;
2366                 }
2367         } else {
2368                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2369                 sc->cfcsum = 0;
2370         }
2371
2372 done:
2373         free(card_fw, M_CXGBE);
2374         if (fw != NULL)
2375                 firmware_put(fw, FIRMWARE_UNLOAD);
2376         if (default_cfg != NULL)
2377                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2378
2379         return (rc);
2380 }
2381
2382 #define FW_PARAM_DEV(param) \
2383         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2384          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2385 #define FW_PARAM_PFVF(param) \
2386         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2387          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2388
2389 /*
2390  * Partition chip resources for use between various PFs, VFs, etc.
2391  */
2392 static int
2393 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2394     const char *name_prefix)
2395 {
2396         const struct firmware *cfg = NULL;
2397         int rc = 0;
2398         struct fw_caps_config_cmd caps;
2399         uint32_t mtype, moff, finicsum, cfcsum;
2400
2401         /*
2402          * Figure out what configuration file to use.  Pick the default config
2403          * file for the card if the user hasn't specified one explicitly.
2404          */
2405         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2406         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2407                 /* Card specific overrides go here. */
2408                 if (pci_get_device(sc->dev) == 0x440a)
2409                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2410                 if (is_fpga(sc))
2411                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2412         }
2413
2414         /*
2415          * We need to load another module if the profile is anything except
2416          * "default" or "flash".
2417          */
2418         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2419             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2420                 char s[32];
2421
2422                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2423                 cfg = firmware_get(s);
2424                 if (cfg == NULL) {
2425                         if (default_cfg != NULL) {
2426                                 device_printf(sc->dev,
2427                                     "unable to load module \"%s\" for "
2428                                     "configuration profile \"%s\", will use "
2429                                     "the default config file instead.\n",
2430                                     s, sc->cfg_file);
2431                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2432                                     "%s", DEFAULT_CF);
2433                         } else {
2434                                 device_printf(sc->dev,
2435                                     "unable to load module \"%s\" for "
2436                                     "configuration profile \"%s\", will use "
2437                                     "the config file on the card's flash "
2438                                     "instead.\n", s, sc->cfg_file);
2439                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2440                                     "%s", FLASH_CF);
2441                         }
2442                 }
2443         }
2444
2445         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2446             default_cfg == NULL) {
2447                 device_printf(sc->dev,
2448                     "default config file not available, will use the config "
2449                     "file on the card's flash instead.\n");
2450                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2451         }
2452
2453         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2454                 u_int cflen, i, n;
2455                 const uint32_t *cfdata;
2456                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2457
2458                 KASSERT(cfg != NULL || default_cfg != NULL,
2459                     ("%s: no config to upload", __func__));
2460
2461                 /*
2462                  * Ask the firmware where it wants us to upload the config file.
2463                  */
2464                 param = FW_PARAM_DEV(CF);
2465                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2466                 if (rc != 0) {
2467                         /* No support for config file?  Shouldn't happen. */
2468                         device_printf(sc->dev,
2469                             "failed to query config file location: %d.\n", rc);
2470                         goto done;
2471                 }
2472                 mtype = G_FW_PARAMS_PARAM_Y(val);
2473                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2474
2475                 /*
2476                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2477                  * useless stuffing/comments at the end of the config file so
2478                  * it's ok to simply throw away the last remaining bytes when
2479                  * the config file is not an exact multiple of 4.  This also
2480                  * helps with the validate_mt_off_len check.
2481                  */
2482                 if (cfg != NULL) {
2483                         cflen = cfg->datasize & ~3;
2484                         cfdata = cfg->data;
2485                 } else {
2486                         cflen = default_cfg->datasize & ~3;
2487                         cfdata = default_cfg->data;
2488                 }
2489
2490                 if (cflen > FLASH_CFG_MAX_SIZE) {
2491                         device_printf(sc->dev,
2492                             "config file too long (%d, max allowed is %d).  "
2493                             "Will try to use the config on the card, if any.\n",
2494                             cflen, FLASH_CFG_MAX_SIZE);
2495                         goto use_config_on_flash;
2496                 }
2497
2498                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2499                 if (rc != 0) {
2500                         device_printf(sc->dev,
2501                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2502                             "Will try to use the config on the card, if any.\n",
2503                             __func__, mtype, moff, cflen, rc);
2504                         goto use_config_on_flash;
2505                 }
2506
2507                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2508                 while (cflen) {
2509                         off = position_memwin(sc, 2, addr);
2510                         n = min(cflen, mw_aperture - off);
2511                         for (i = 0; i < n; i += 4)
2512                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2513                         cflen -= n;
2514                         addr += n;
2515                 }
2516         } else {
2517 use_config_on_flash:
2518                 mtype = FW_MEMTYPE_FLASH;
2519                 moff = t4_flash_cfg_addr(sc);
2520         }
2521
2522         bzero(&caps, sizeof(caps));
2523         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2524             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2525         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2526             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2527             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2528         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2529         if (rc != 0) {
2530                 device_printf(sc->dev,
2531                     "failed to pre-process config file: %d "
2532                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2533                 goto done;
2534         }
2535
2536         finicsum = be32toh(caps.finicsum);
2537         cfcsum = be32toh(caps.cfcsum);
2538         if (finicsum != cfcsum) {
2539                 device_printf(sc->dev,
2540                     "WARNING: config file checksum mismatch: %08x %08x\n",
2541                     finicsum, cfcsum);
2542         }
2543         sc->cfcsum = cfcsum;
2544
2545 #define LIMIT_CAPS(x) do { \
2546         caps.x &= htobe16(t4_##x##_allowed); \
2547 } while (0)
2548
2549         /*
2550          * Let the firmware know what features will (not) be used so it can tune
2551          * things accordingly.
2552          */
2553         LIMIT_CAPS(linkcaps);
2554         LIMIT_CAPS(niccaps);
2555         LIMIT_CAPS(toecaps);
2556         LIMIT_CAPS(rdmacaps);
2557         LIMIT_CAPS(iscsicaps);
2558         LIMIT_CAPS(fcoecaps);
2559 #undef LIMIT_CAPS
2560
2561         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2562             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2563         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2564         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2565         if (rc != 0) {
2566                 device_printf(sc->dev,
2567                     "failed to process config file: %d.\n", rc);
2568         }
2569 done:
2570         if (cfg != NULL)
2571                 firmware_put(cfg, FIRMWARE_UNLOAD);
2572         return (rc);
2573 }
2574
2575 /*
2576  * Retrieve parameters that are needed (or nice to have) very early.
2577  */
2578 static int
2579 get_params__pre_init(struct adapter *sc)
2580 {
2581         int rc;
2582         uint32_t param[2], val[2];
2583         struct fw_devlog_cmd cmd;
2584         struct devlog_params *dlog = &sc->params.devlog;
2585
2586         param[0] = FW_PARAM_DEV(PORTVEC);
2587         param[1] = FW_PARAM_DEV(CCLK);
2588         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2589         if (rc != 0) {
2590                 device_printf(sc->dev,
2591                     "failed to query parameters (pre_init): %d.\n", rc);
2592                 return (rc);
2593         }
2594
2595         sc->params.portvec = val[0];
2596         sc->params.nports = bitcount32(val[0]);
2597         sc->params.vpd.cclk = val[1];
2598
2599         /* Read device log parameters. */
2600         bzero(&cmd, sizeof(cmd));
2601         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2602             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2603         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2604         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2605         if (rc != 0) {
2606                 device_printf(sc->dev,
2607                     "failed to get devlog parameters: %d.\n", rc);
2608                 bzero(dlog, sizeof (*dlog));
2609                 rc = 0; /* devlog isn't critical for device operation */
2610         } else {
2611                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2612                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2613                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2614                 dlog->size = be32toh(cmd.memsize_devlog);
2615         }
2616
2617         return (rc);
2618 }
2619
2620 /*
2621  * Retrieve various parameters that are of interest to the driver.  The device
2622  * has been initialized by the firmware at this point.
2623  */
2624 static int
2625 get_params__post_init(struct adapter *sc)
2626 {
2627         int rc;
2628         uint32_t param[7], val[7];
2629         struct fw_caps_config_cmd caps;
2630
2631         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2632         param[1] = FW_PARAM_PFVF(EQ_START);
2633         param[2] = FW_PARAM_PFVF(FILTER_START);
2634         param[3] = FW_PARAM_PFVF(FILTER_END);
2635         param[4] = FW_PARAM_PFVF(L2T_START);
2636         param[5] = FW_PARAM_PFVF(L2T_END);
2637         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2638         if (rc != 0) {
2639                 device_printf(sc->dev,
2640                     "failed to query parameters (post_init): %d.\n", rc);
2641                 return (rc);
2642         }
2643
2644         sc->sge.iq_start = val[0];
2645         sc->sge.eq_start = val[1];
2646         sc->tids.ftid_base = val[2];
2647         sc->tids.nftids = val[3] - val[2] + 1;
2648         sc->params.ftid_min = val[2];
2649         sc->params.ftid_max = val[3];
2650         sc->vres.l2t.start = val[4];
2651         sc->vres.l2t.size = val[5] - val[4] + 1;
2652         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2653             ("%s: L2 table size (%u) larger than expected (%u)",
2654             __func__, sc->vres.l2t.size, L2T_SIZE));
2655
2656         /* get capabilites */
2657         bzero(&caps, sizeof(caps));
2658         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2659             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2660         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2661         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2662         if (rc != 0) {
2663                 device_printf(sc->dev,
2664                     "failed to get card capabilities: %d.\n", rc);
2665                 return (rc);
2666         }
2667
2668 #define READ_CAPS(x) do { \
2669         sc->x = htobe16(caps.x); \
2670 } while (0)
2671         READ_CAPS(linkcaps);
2672         READ_CAPS(niccaps);
2673         READ_CAPS(toecaps);
2674         READ_CAPS(rdmacaps);
2675         READ_CAPS(iscsicaps);
2676         READ_CAPS(fcoecaps);
2677
2678         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2679                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2680                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2681                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2682                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2683                 if (rc != 0) {
2684                         device_printf(sc->dev,
2685                             "failed to query NIC parameters: %d.\n", rc);
2686                         return (rc);
2687                 }
2688                 sc->tids.etid_base = val[0];
2689                 sc->params.etid_min = val[0];
2690                 sc->tids.netids = val[1] - val[0] + 1;
2691                 sc->params.netids = sc->tids.netids;
2692                 sc->params.eo_wr_cred = val[2];
2693                 sc->params.ethoffload = 1;
2694         }
2695
2696         if (sc->toecaps) {
2697                 /* query offload-related parameters */
2698                 param[0] = FW_PARAM_DEV(NTID);
2699                 param[1] = FW_PARAM_PFVF(SERVER_START);
2700                 param[2] = FW_PARAM_PFVF(SERVER_END);
2701                 param[3] = FW_PARAM_PFVF(TDDP_START);
2702                 param[4] = FW_PARAM_PFVF(TDDP_END);
2703                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2704                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2705                 if (rc != 0) {
2706                         device_printf(sc->dev,
2707                             "failed to query TOE parameters: %d.\n", rc);
2708                         return (rc);
2709                 }
2710                 sc->tids.ntids = val[0];
2711                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2712                 sc->tids.stid_base = val[1];
2713                 sc->tids.nstids = val[2] - val[1] + 1;
2714                 sc->vres.ddp.start = val[3];
2715                 sc->vres.ddp.size = val[4] - val[3] + 1;
2716                 sc->params.ofldq_wr_cred = val[5];
2717                 sc->params.offload = 1;
2718         }
2719         if (sc->rdmacaps) {
2720                 param[0] = FW_PARAM_PFVF(STAG_START);
2721                 param[1] = FW_PARAM_PFVF(STAG_END);
2722                 param[2] = FW_PARAM_PFVF(RQ_START);
2723                 param[3] = FW_PARAM_PFVF(RQ_END);
2724                 param[4] = FW_PARAM_PFVF(PBL_START);
2725                 param[5] = FW_PARAM_PFVF(PBL_END);
2726                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2727                 if (rc != 0) {
2728                         device_printf(sc->dev,
2729                             "failed to query RDMA parameters(1): %d.\n", rc);
2730                         return (rc);
2731                 }
2732                 sc->vres.stag.start = val[0];
2733                 sc->vres.stag.size = val[1] - val[0] + 1;
2734                 sc->vres.rq.start = val[2];
2735                 sc->vres.rq.size = val[3] - val[2] + 1;
2736                 sc->vres.pbl.start = val[4];
2737                 sc->vres.pbl.size = val[5] - val[4] + 1;
2738
2739                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2740                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2741                 param[2] = FW_PARAM_PFVF(CQ_START);
2742                 param[3] = FW_PARAM_PFVF(CQ_END);
2743                 param[4] = FW_PARAM_PFVF(OCQ_START);
2744                 param[5] = FW_PARAM_PFVF(OCQ_END);
2745                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2746                 if (rc != 0) {
2747                         device_printf(sc->dev,
2748                             "failed to query RDMA parameters(2): %d.\n", rc);
2749                         return (rc);
2750                 }
2751                 sc->vres.qp.start = val[0];
2752                 sc->vres.qp.size = val[1] - val[0] + 1;
2753                 sc->vres.cq.start = val[2];
2754                 sc->vres.cq.size = val[3] - val[2] + 1;
2755                 sc->vres.ocq.start = val[4];
2756                 sc->vres.ocq.size = val[5] - val[4] + 1;
2757         }
2758         if (sc->iscsicaps) {
2759                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2760                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2761                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2762                 if (rc != 0) {
2763                         device_printf(sc->dev,
2764                             "failed to query iSCSI parameters: %d.\n", rc);
2765                         return (rc);
2766                 }
2767                 sc->vres.iscsi.start = val[0];
2768                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2769         }
2770
2771         /*
2772          * We've got the params we wanted to query via the firmware.  Now grab
2773          * some others directly from the chip.
2774          */
2775         rc = t4_read_chip_settings(sc);
2776
2777         return (rc);
2778 }
2779
2780 static int
2781 set_params__post_init(struct adapter *sc)
2782 {
2783         uint32_t param, val;
2784
2785         /* ask for encapsulated CPLs */
2786         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2787         val = 1;
2788         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2789
2790         return (0);
2791 }
2792
2793 #undef FW_PARAM_PFVF
2794 #undef FW_PARAM_DEV
2795
2796 static void
2797 t4_set_desc(struct adapter *sc)
2798 {
2799         char buf[128];
2800         struct adapter_params *p = &sc->params;
2801
2802         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2803             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2804             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2805
2806         device_set_desc_copy(sc->dev, buf);
2807 }
2808
2809 static void
2810 build_medialist(struct port_info *pi, struct ifmedia *media)
2811 {
2812         int data, m;
2813
2814         PORT_LOCK(pi);
2815
2816         ifmedia_removeall(media);
2817
2818         m = IFM_ETHER | IFM_FDX;
2819         data = (pi->port_type << 8) | pi->mod_type;
2820
2821         switch(pi->port_type) {
2822         case FW_PORT_TYPE_BT_XFI:
2823         case FW_PORT_TYPE_BT_XAUI:
2824                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2825                 /* fall through */
2826
2827         case FW_PORT_TYPE_BT_SGMII:
2828                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2829                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2830                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2831                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2832                 break;
2833
2834         case FW_PORT_TYPE_CX4:
2835                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2836                 ifmedia_set(media, m | IFM_10G_CX4);
2837                 break;
2838
2839         case FW_PORT_TYPE_QSFP_10G:
2840         case FW_PORT_TYPE_SFP:
2841         case FW_PORT_TYPE_FIBER_XFI:
2842         case FW_PORT_TYPE_FIBER_XAUI:
2843                 switch (pi->mod_type) {
2844
2845                 case FW_PORT_MOD_TYPE_LR:
2846                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2847                         ifmedia_set(media, m | IFM_10G_LR);
2848                         break;
2849
2850                 case FW_PORT_MOD_TYPE_SR:
2851                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2852                         ifmedia_set(media, m | IFM_10G_SR);
2853                         break;
2854
2855                 case FW_PORT_MOD_TYPE_LRM:
2856                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2857                         ifmedia_set(media, m | IFM_10G_LRM);
2858                         break;
2859
2860                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2861                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2862                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2863                         ifmedia_set(media, m | IFM_10G_TWINAX);
2864                         break;
2865
2866                 case FW_PORT_MOD_TYPE_NONE:
2867                         m &= ~IFM_FDX;
2868                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2869                         ifmedia_set(media, m | IFM_NONE);
2870                         break;
2871
2872                 case FW_PORT_MOD_TYPE_NA:
2873                 case FW_PORT_MOD_TYPE_ER:
2874                 default:
2875                         device_printf(pi->dev,
2876                             "unknown port_type (%d), mod_type (%d)\n",
2877                             pi->port_type, pi->mod_type);
2878                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2879                         ifmedia_set(media, m | IFM_UNKNOWN);
2880                         break;
2881                 }
2882                 break;
2883
2884         case FW_PORT_TYPE_QSFP:
2885                 switch (pi->mod_type) {
2886
2887                 case FW_PORT_MOD_TYPE_LR:
2888                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2889                         ifmedia_set(media, m | IFM_40G_LR4);
2890                         break;
2891
2892                 case FW_PORT_MOD_TYPE_SR:
2893                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2894                         ifmedia_set(media, m | IFM_40G_SR4);
2895                         break;
2896
2897                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2898                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2899                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2900                         ifmedia_set(media, m | IFM_40G_CR4);
2901                         break;
2902
2903                 case FW_PORT_MOD_TYPE_NONE:
2904                         m &= ~IFM_FDX;
2905                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2906                         ifmedia_set(media, m | IFM_NONE);
2907                         break;
2908
2909                 default:
2910                         device_printf(pi->dev,
2911                             "unknown port_type (%d), mod_type (%d)\n",
2912                             pi->port_type, pi->mod_type);
2913                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2914                         ifmedia_set(media, m | IFM_UNKNOWN);
2915                         break;
2916                 }
2917                 break;
2918
2919         default:
2920                 device_printf(pi->dev,
2921                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2922                     pi->mod_type);
2923                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2924                 ifmedia_set(media, m | IFM_UNKNOWN);
2925                 break;
2926         }
2927
2928         PORT_UNLOCK(pi);
2929 }
2930
2931 #define FW_MAC_EXACT_CHUNK      7
2932
2933 /*
2934  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2935  * indicates which parameters should be programmed (the rest are left alone).
2936  */
2937 int
2938 update_mac_settings(struct ifnet *ifp, int flags)
2939 {
2940         int rc = 0;
2941         struct port_info *pi = ifp->if_softc;
2942         struct adapter *sc = pi->adapter;
2943         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2944         uint16_t viid = 0xffff;
2945         int16_t *xact_addr_filt = NULL;
2946
2947         ASSERT_SYNCHRONIZED_OP(sc);
2948         KASSERT(flags, ("%s: not told what to update.", __func__));
2949
2950         if (ifp == pi->ifp) {
2951                 viid = pi->viid;
2952                 xact_addr_filt = &pi->xact_addr_filt;
2953         }
2954 #ifdef DEV_NETMAP
2955         else if (ifp == pi->nm_ifp) {
2956                 viid = pi->nm_viid;
2957                 xact_addr_filt = &pi->nm_xact_addr_filt;
2958         }
2959 #endif
2960         if (flags & XGMAC_MTU)
2961                 mtu = ifp->if_mtu;
2962
2963         if (flags & XGMAC_PROMISC)
2964                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2965
2966         if (flags & XGMAC_ALLMULTI)
2967                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2968
2969         if (flags & XGMAC_VLANEX)
2970                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2971
2972         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2973                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2974                     1, vlanex, false);
2975                 if (rc) {
2976                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2977                             rc);
2978                         return (rc);
2979                 }
2980         }
2981
2982         if (flags & XGMAC_UCADDR) {
2983                 uint8_t ucaddr[ETHER_ADDR_LEN];
2984
2985                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2986                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2987                     true, true);
2988                 if (rc < 0) {
2989                         rc = -rc;
2990                         if_printf(ifp, "change_mac failed: %d\n", rc);
2991                         return (rc);
2992                 } else {
2993                         *xact_addr_filt = rc;
2994                         rc = 0;
2995                 }
2996         }
2997
2998         if (flags & XGMAC_MCADDRS) {
2999                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3000                 int del = 1;
3001                 uint64_t hash = 0;
3002                 struct ifmultiaddr *ifma;
3003                 int i = 0, j;
3004
3005                 if_maddr_rlock(ifp);
3006                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3007                         if (ifma->ifma_addr->sa_family != AF_LINK)
3008                                 continue;
3009                         mcaddr[i] =
3010                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3011                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3012                         i++;
3013
3014                         if (i == FW_MAC_EXACT_CHUNK) {
3015                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3016                                     i, mcaddr, NULL, &hash, 0);
3017                                 if (rc < 0) {
3018                                         rc = -rc;
3019                                         for (j = 0; j < i; j++) {
3020                                                 if_printf(ifp,
3021                                                     "failed to add mc address"
3022                                                     " %02x:%02x:%02x:"
3023                                                     "%02x:%02x:%02x rc=%d\n",
3024                                                     mcaddr[j][0], mcaddr[j][1],
3025                                                     mcaddr[j][2], mcaddr[j][3],
3026                                                     mcaddr[j][4], mcaddr[j][5],
3027                                                     rc);
3028                                         }
3029                                         goto mcfail;
3030                                 }
3031                                 del = 0;
3032                                 i = 0;
3033                         }
3034                 }
3035                 if (i > 0) {
3036                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3037                             mcaddr, NULL, &hash, 0);
3038                         if (rc < 0) {
3039                                 rc = -rc;
3040                                 for (j = 0; j < i; j++) {
3041                                         if_printf(ifp,
3042                                             "failed to add mc address"
3043                                             " %02x:%02x:%02x:"
3044                                             "%02x:%02x:%02x rc=%d\n",
3045                                             mcaddr[j][0], mcaddr[j][1],
3046                                             mcaddr[j][2], mcaddr[j][3],
3047                                             mcaddr[j][4], mcaddr[j][5],
3048                                             rc);
3049                                 }
3050                                 goto mcfail;
3051                         }
3052                 }
3053
3054                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3055                 if (rc != 0)
3056                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3057 mcfail:
3058                 if_maddr_runlock(ifp);
3059         }
3060
3061         return (rc);
3062 }
3063
3064 /*
3065  * {begin|end}_synchronized_op must be called from the same thread.
3066  */
3067 int
3068 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3069     char *wmesg)
3070 {
3071         int rc, pri;
3072
3073 #ifdef WITNESS
3074         /* the caller thinks it's ok to sleep, but is it really? */
3075         if (flags & SLEEP_OK)
3076                 pause("t4slptst", 1);
3077 #endif
3078
3079         if (INTR_OK)
3080                 pri = PCATCH;
3081         else
3082                 pri = 0;
3083
3084         ADAPTER_LOCK(sc);
3085         for (;;) {
3086
3087                 if (pi && IS_DOOMED(pi)) {
3088                         rc = ENXIO;
3089                         goto done;
3090                 }
3091
3092                 if (!IS_BUSY(sc)) {
3093                         rc = 0;
3094                         break;
3095                 }
3096
3097                 if (!(flags & SLEEP_OK)) {
3098                         rc = EBUSY;
3099                         goto done;
3100                 }
3101
3102                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3103                         rc = EINTR;
3104                         goto done;
3105                 }
3106         }
3107
3108         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3109         SET_BUSY(sc);
3110 #ifdef INVARIANTS
3111         sc->last_op = wmesg;
3112         sc->last_op_thr = curthread;
3113         sc->last_op_flags = flags;
3114 #endif
3115
3116 done:
3117         if (!(flags & HOLD_LOCK) || rc)
3118                 ADAPTER_UNLOCK(sc);
3119
3120         return (rc);
3121 }
3122
3123 /*
3124  * {begin|end}_synchronized_op must be called from the same thread.
3125  */
3126 void
3127 end_synchronized_op(struct adapter *sc, int flags)
3128 {
3129
3130         if (flags & LOCK_HELD)
3131                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3132         else
3133                 ADAPTER_LOCK(sc);
3134
3135         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3136         CLR_BUSY(sc);
3137         wakeup(&sc->flags);
3138         ADAPTER_UNLOCK(sc);
3139 }
3140
3141 static int
3142 cxgbe_init_synchronized(struct port_info *pi)
3143 {
3144         struct adapter *sc = pi->adapter;
3145         struct ifnet *ifp = pi->ifp;
3146         int rc = 0, i;
3147         struct sge_txq *txq;
3148
3149         ASSERT_SYNCHRONIZED_OP(sc);
3150
3151         if (isset(&sc->open_device_map, pi->port_id)) {
3152                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3153                     ("mismatch between open_device_map and if_drv_flags"));
3154                 return (0);     /* already running */
3155         }
3156
3157         if (!(sc->flags & FULL_INIT_DONE) &&
3158             ((rc = adapter_full_init(sc)) != 0))
3159                 return (rc);    /* error message displayed already */
3160
3161         if (!(pi->flags & PORT_INIT_DONE) &&
3162             ((rc = port_full_init(pi)) != 0))
3163                 return (rc); /* error message displayed already */
3164
3165         rc = update_mac_settings(ifp, XGMAC_ALL);
3166         if (rc)
3167                 goto done;      /* error message displayed already */
3168
3169         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3170         if (rc != 0) {
3171                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3172                 goto done;
3173         }
3174
3175         /*
3176          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3177          * if this changes.
3178          */
3179
3180         for_each_txq(pi, i, txq) {
3181                 TXQ_LOCK(txq);
3182                 txq->eq.flags |= EQ_ENABLED;
3183                 TXQ_UNLOCK(txq);
3184         }
3185
3186         /*
3187          * The first iq of the first port to come up is used for tracing.
3188          */
3189         if (sc->traceq < 0) {
3190                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3191                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3192                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3193                     V_QUEUENUMBER(sc->traceq));
3194                 pi->flags |= HAS_TRACEQ;
3195         }
3196
3197         /* all ok */
3198         setbit(&sc->open_device_map, pi->port_id);
3199         PORT_LOCK(pi);
3200         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3201         PORT_UNLOCK(pi);
3202
3203         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3204 done:
3205         if (rc != 0)
3206                 cxgbe_uninit_synchronized(pi);
3207
3208         return (rc);
3209 }
3210
3211 /*
3212  * Idempotent.
3213  */
3214 static int
3215 cxgbe_uninit_synchronized(struct port_info *pi)
3216 {
3217         struct adapter *sc = pi->adapter;
3218         struct ifnet *ifp = pi->ifp;
3219         int rc, i;
3220         struct sge_txq *txq;
3221
3222         ASSERT_SYNCHRONIZED_OP(sc);
3223
3224         /*
3225          * Disable the VI so that all its data in either direction is discarded
3226          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3227          * tick) intact as the TP can deliver negative advice or data that it's
3228          * holding in its RAM (for an offloaded connection) even after the VI is
3229          * disabled.
3230          */
3231         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3232         if (rc) {
3233                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3234                 return (rc);
3235         }
3236
3237         for_each_txq(pi, i, txq) {
3238                 TXQ_LOCK(txq);
3239                 txq->eq.flags &= ~EQ_ENABLED;
3240                 TXQ_UNLOCK(txq);
3241         }
3242
3243         clrbit(&sc->open_device_map, pi->port_id);
3244         PORT_LOCK(pi);
3245         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3246         PORT_UNLOCK(pi);
3247
3248         pi->link_cfg.link_ok = 0;
3249         pi->link_cfg.speed = 0;
3250         pi->linkdnrc = -1;
3251         t4_os_link_changed(sc, pi->port_id, 0, -1);
3252
3253         return (0);
3254 }
3255
3256 /*
3257  * It is ok for this function to fail midway and return right away.  t4_detach
3258  * will walk the entire sc->irq list and clean up whatever is valid.
3259  */
3260 static int
3261 setup_intr_handlers(struct adapter *sc)
3262 {
3263         int rc, rid, p, q;
3264         char s[8];
3265         struct irq *irq;
3266         struct port_info *pi;
3267         struct sge_rxq *rxq;
3268 #ifdef TCP_OFFLOAD
3269         struct sge_ofld_rxq *ofld_rxq;
3270 #endif
3271 #ifdef DEV_NETMAP
3272         struct sge_nm_rxq *nm_rxq;
3273 #endif
3274
3275         /*
3276          * Setup interrupts.
3277          */
3278         irq = &sc->irq[0];
3279         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3280         if (sc->intr_count == 1)
3281                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3282
3283         /* Multiple interrupts. */
3284         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3285             ("%s: too few intr.", __func__));
3286
3287         /* The first one is always error intr */
3288         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3289         if (rc != 0)
3290                 return (rc);
3291         irq++;
3292         rid++;
3293
3294         /* The second one is always the firmware event queue */
3295         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3296         if (rc != 0)
3297                 return (rc);
3298         irq++;
3299         rid++;
3300
3301         for_each_port(sc, p) {
3302                 pi = sc->port[p];
3303
3304                 if (pi->flags & INTR_RXQ) {
3305                         for_each_rxq(pi, q, rxq) {
3306                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3307                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3308                                     s);
3309                                 if (rc != 0)
3310                                         return (rc);
3311                                 irq++;
3312                                 rid++;
3313                         }
3314                 }
3315 #ifdef TCP_OFFLOAD
3316                 if (pi->flags & INTR_OFLD_RXQ) {
3317                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3318                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3319                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3320                                     ofld_rxq, s);
3321                                 if (rc != 0)
3322                                         return (rc);
3323                                 irq++;
3324                                 rid++;
3325                         }
3326                 }
3327 #endif
3328 #ifdef DEV_NETMAP
3329                 if (pi->flags & INTR_NM_RXQ) {
3330                         for_each_nm_rxq(pi, q, nm_rxq) {
3331                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3332                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3333                                     nm_rxq, s);
3334                                 if (rc != 0)
3335                                         return (rc);
3336                                 irq++;
3337                                 rid++;
3338                         }
3339                 }
3340 #endif
3341         }
3342         MPASS(irq == &sc->irq[sc->intr_count]);
3343
3344         return (0);
3345 }
3346
3347 int
3348 adapter_full_init(struct adapter *sc)
3349 {
3350         int rc, i;
3351
3352         ASSERT_SYNCHRONIZED_OP(sc);
3353         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3354         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3355             ("%s: FULL_INIT_DONE already", __func__));
3356
3357         /*
3358          * queues that belong to the adapter (not any particular port).
3359          */
3360         rc = t4_setup_adapter_queues(sc);
3361         if (rc != 0)
3362                 goto done;
3363
3364         for (i = 0; i < nitems(sc->tq); i++) {
3365                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3366                     taskqueue_thread_enqueue, &sc->tq[i]);
3367                 if (sc->tq[i] == NULL) {
3368                         device_printf(sc->dev,
3369                             "failed to allocate task queue %d\n", i);
3370                         rc = ENOMEM;
3371                         goto done;
3372                 }
3373                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3374                     device_get_nameunit(sc->dev), i);
3375         }
3376
3377         t4_intr_enable(sc);
3378         sc->flags |= FULL_INIT_DONE;
3379 done:
3380         if (rc != 0)
3381                 adapter_full_uninit(sc);
3382
3383         return (rc);
3384 }
3385
3386 int
3387 adapter_full_uninit(struct adapter *sc)
3388 {
3389         int i;
3390
3391         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3392
3393         t4_teardown_adapter_queues(sc);
3394
3395         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3396                 taskqueue_free(sc->tq[i]);
3397                 sc->tq[i] = NULL;
3398         }
3399
3400         sc->flags &= ~FULL_INIT_DONE;
3401
3402         return (0);
3403 }
3404
3405 int
3406 port_full_init(struct port_info *pi)
3407 {
3408         struct adapter *sc = pi->adapter;
3409         struct ifnet *ifp = pi->ifp;
3410         uint16_t *rss;
3411         struct sge_rxq *rxq;
3412         int rc, i, j;
3413
3414         ASSERT_SYNCHRONIZED_OP(sc);
3415         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3416             ("%s: PORT_INIT_DONE already", __func__));
3417
3418         sysctl_ctx_init(&pi->ctx);
3419         pi->flags |= PORT_SYSCTL_CTX;
3420
3421         /*
3422          * Allocate tx/rx/fl queues for this port.
3423          */
3424         rc = t4_setup_port_queues(pi);
3425         if (rc != 0)
3426                 goto done;      /* error message displayed already */
3427
3428         /*
3429          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3430          */
3431         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3432         for (i = 0; i < pi->rss_size;) {
3433                 for_each_rxq(pi, j, rxq) {
3434                         rss[i++] = rxq->iq.abs_id;
3435                         if (i == pi->rss_size)
3436                                 break;
3437                 }
3438         }
3439
3440         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3441             pi->rss_size);
3442         if (rc != 0) {
3443                 if_printf(ifp, "rss_config failed: %d\n", rc);
3444                 goto done;
3445         }
3446
3447         pi->rss = rss;
3448         pi->flags |= PORT_INIT_DONE;
3449 done:
3450         if (rc != 0)
3451                 port_full_uninit(pi);
3452
3453         return (rc);
3454 }
3455
3456 /*
3457  * Idempotent.
3458  */
3459 int
3460 port_full_uninit(struct port_info *pi)
3461 {
3462         struct adapter *sc = pi->adapter;
3463         int i;
3464         struct sge_rxq *rxq;
3465         struct sge_txq *txq;
3466 #ifdef TCP_OFFLOAD
3467         struct sge_ofld_rxq *ofld_rxq;
3468         struct sge_wrq *ofld_txq;
3469 #endif
3470
3471         if (pi->flags & PORT_INIT_DONE) {
3472
3473                 /* Need to quiesce queues.  */
3474
3475                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3476
3477                 for_each_txq(pi, i, txq) {
3478                         quiesce_txq(sc, txq);
3479                 }
3480
3481 #ifdef TCP_OFFLOAD
3482                 for_each_ofld_txq(pi, i, ofld_txq) {
3483                         quiesce_wrq(sc, ofld_txq);
3484                 }
3485 #endif
3486
3487                 for_each_rxq(pi, i, rxq) {
3488                         quiesce_iq(sc, &rxq->iq);
3489                         quiesce_fl(sc, &rxq->fl);
3490                 }
3491
3492 #ifdef TCP_OFFLOAD
3493                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3494                         quiesce_iq(sc, &ofld_rxq->iq);
3495                         quiesce_fl(sc, &ofld_rxq->fl);
3496                 }
3497 #endif
3498                 free(pi->rss, M_CXGBE);
3499         }
3500
3501         t4_teardown_port_queues(pi);
3502         pi->flags &= ~PORT_INIT_DONE;
3503
3504         return (0);
3505 }
3506
3507 static void
3508 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3509 {
3510         struct sge_eq *eq = &txq->eq;
3511         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3512
3513         (void) sc;      /* unused */
3514
3515 #ifdef INVARIANTS
3516         TXQ_LOCK(txq);
3517         MPASS((eq->flags & EQ_ENABLED) == 0);
3518         TXQ_UNLOCK(txq);
3519 #endif
3520
3521         /* Wait for the mp_ring to empty. */
3522         while (!mp_ring_is_idle(txq->r)) {
3523                 mp_ring_check_drainage(txq->r, 0);
3524                 pause("rquiesce", 1);
3525         }
3526
3527         /* Then wait for the hardware to finish. */
3528         while (spg->cidx != htobe16(eq->pidx))
3529                 pause("equiesce", 1);
3530
3531         /* Finally, wait for the driver to reclaim all descriptors. */
3532         while (eq->cidx != eq->pidx)
3533                 pause("dquiesce", 1);
3534 }
3535
3536 static void
3537 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3538 {
3539
3540         /* XXXTX */
3541 }
3542
3543 static void
3544 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3545 {
3546         (void) sc;      /* unused */
3547
3548         /* Synchronize with the interrupt handler */
3549         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3550                 pause("iqfree", 1);
3551 }
3552
3553 static void
3554 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3555 {
3556         mtx_lock(&sc->sfl_lock);
3557         FL_LOCK(fl);
3558         fl->flags |= FL_DOOMED;
3559         FL_UNLOCK(fl);
3560         mtx_unlock(&sc->sfl_lock);
3561
3562         callout_drain(&sc->sfl_callout);
3563         KASSERT((fl->flags & FL_STARVING) == 0,
3564             ("%s: still starving", __func__));
3565 }
3566
3567 static int
3568 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3569     driver_intr_t *handler, void *arg, char *name)
3570 {
3571         int rc;
3572
3573         irq->rid = rid;
3574         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3575             RF_SHAREABLE | RF_ACTIVE);
3576         if (irq->res == NULL) {
3577                 device_printf(sc->dev,
3578                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3579                 return (ENOMEM);
3580         }
3581
3582         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3583             NULL, handler, arg, &irq->tag);
3584         if (rc != 0) {
3585                 device_printf(sc->dev,
3586                     "failed to setup interrupt for rid %d, name %s: %d\n",
3587                     rid, name, rc);
3588         } else if (name)
3589                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3590
3591         return (rc);
3592 }
3593
3594 static int
3595 t4_free_irq(struct adapter *sc, struct irq *irq)
3596 {
3597         if (irq->tag)
3598                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3599         if (irq->res)
3600                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3601
3602         bzero(irq, sizeof(*irq));
3603
3604         return (0);
3605 }
3606
3607 static void
3608 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3609     unsigned int end)
3610 {
3611         uint32_t *p = (uint32_t *)(buf + start);
3612
3613         for ( ; start <= end; start += sizeof(uint32_t))
3614                 *p++ = t4_read_reg(sc, start);
3615 }
3616
3617 static void
3618 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3619 {
3620         int i, n;
3621         const unsigned int *reg_ranges;
3622         static const unsigned int t4_reg_ranges[] = {
3623                 0x1008, 0x1108,
3624                 0x1180, 0x11b4,
3625                 0x11fc, 0x123c,
3626                 0x1300, 0x173c,
3627                 0x1800, 0x18fc,
3628                 0x3000, 0x30d8,
3629                 0x30e0, 0x5924,
3630                 0x5960, 0x59d4,
3631                 0x5a00, 0x5af8,
3632                 0x6000, 0x6098,
3633                 0x6100, 0x6150,
3634                 0x6200, 0x6208,
3635                 0x6240, 0x6248,
3636                 0x6280, 0x6338,
3637                 0x6370, 0x638c,
3638                 0x6400, 0x643c,
3639                 0x6500, 0x6524,
3640                 0x6a00, 0x6a38,
3641                 0x6a60, 0x6a78,
3642                 0x6b00, 0x6b84,
3643                 0x6bf0, 0x6c84,
3644                 0x6cf0, 0x6d84,
3645                 0x6df0, 0x6e84,
3646                 0x6ef0, 0x6f84,
3647                 0x6ff0, 0x7084,
3648                 0x70f0, 0x7184,
3649                 0x71f0, 0x7284,
3650                 0x72f0, 0x7384,
3651                 0x73f0, 0x7450,
3652                 0x7500, 0x7530,
3653                 0x7600, 0x761c,
3654                 0x7680, 0x76cc,
3655                 0x7700, 0x7798,
3656                 0x77c0, 0x77fc,
3657                 0x7900, 0x79fc,
3658                 0x7b00, 0x7c38,
3659                 0x7d00, 0x7efc,
3660                 0x8dc0, 0x8e1c,
3661                 0x8e30, 0x8e78,
3662                 0x8ea0, 0x8f6c,
3663                 0x8fc0, 0x9074,
3664                 0x90fc, 0x90fc,
3665                 0x9400, 0x9458,
3666                 0x9600, 0x96bc,
3667                 0x9800, 0x9808,
3668                 0x9820, 0x983c,
3669                 0x9850, 0x9864,
3670                 0x9c00, 0x9c6c,
3671                 0x9c80, 0x9cec,
3672                 0x9d00, 0x9d6c,
3673                 0x9d80, 0x9dec,
3674                 0x9e00, 0x9e6c,
3675                 0x9e80, 0x9eec,
3676                 0x9f00, 0x9f6c,
3677                 0x9f80, 0x9fec,
3678                 0xd004, 0xd03c,
3679                 0xdfc0, 0xdfe0,
3680                 0xe000, 0xea7c,
3681                 0xf000, 0x11110,
3682                 0x11118, 0x11190,
3683                 0x19040, 0x1906c,
3684                 0x19078, 0x19080,
3685                 0x1908c, 0x19124,
3686                 0x19150, 0x191b0,
3687                 0x191d0, 0x191e8,
3688                 0x19238, 0x1924c,
3689                 0x193f8, 0x19474,
3690                 0x19490, 0x194f8,
3691                 0x19800, 0x19f30,
3692                 0x1a000, 0x1a06c,
3693                 0x1a0b0, 0x1a120,
3694                 0x1a128, 0x1a138,
3695                 0x1a190, 0x1a1c4,
3696                 0x1a1fc, 0x1a1fc,
3697                 0x1e040, 0x1e04c,
3698                 0x1e284, 0x1e28c,
3699                 0x1e2c0, 0x1e2c0,
3700                 0x1e2e0, 0x1e2e0,
3701                 0x1e300, 0x1e384,
3702                 0x1e3c0, 0x1e3c8,
3703                 0x1e440, 0x1e44c,
3704                 0x1e684, 0x1e68c,
3705                 0x1e6c0, 0x1e6c0,
3706                 0x1e6e0, 0x1e6e0,
3707                 0x1e700, 0x1e784,
3708                 0x1e7c0, 0x1e7c8,
3709                 0x1e840, 0x1e84c,
3710                 0x1ea84, 0x1ea8c,
3711                 0x1eac0, 0x1eac0,
3712                 0x1eae0, 0x1eae0,
3713                 0x1eb00, 0x1eb84,
3714                 0x1ebc0, 0x1ebc8,
3715                 0x1ec40, 0x1ec4c,
3716                 0x1ee84, 0x1ee8c,
3717                 0x1eec0, 0x1eec0,
3718                 0x1eee0, 0x1eee0,
3719                 0x1ef00, 0x1ef84,
3720                 0x1efc0, 0x1efc8,
3721                 0x1f040, 0x1f04c,
3722                 0x1f284, 0x1f28c,
3723                 0x1f2c0, 0x1f2c0,
3724                 0x1f2e0, 0x1f2e0,
3725                 0x1f300, 0x1f384,
3726                 0x1f3c0, 0x1f3c8,
3727                 0x1f440, 0x1f44c,
3728                 0x1f684, 0x1f68c,
3729                 0x1f6c0, 0x1f6c0,
3730                 0x1f6e0, 0x1f6e0,
3731                 0x1f700, 0x1f784,
3732                 0x1f7c0, 0x1f7c8,
3733                 0x1f840, 0x1f84c,
3734                 0x1fa84, 0x1fa8c,
3735                 0x1fac0, 0x1fac0,
3736                 0x1fae0, 0x1fae0,
3737                 0x1fb00, 0x1fb84,
3738                 0x1fbc0, 0x1fbc8,
3739                 0x1fc40, 0x1fc4c,
3740                 0x1fe84, 0x1fe8c,
3741                 0x1fec0, 0x1fec0,
3742                 0x1fee0, 0x1fee0,
3743                 0x1ff00, 0x1ff84,
3744                 0x1ffc0, 0x1ffc8,
3745                 0x20000, 0x2002c,
3746                 0x20100, 0x2013c,
3747                 0x20190, 0x201c8,
3748                 0x20200, 0x20318,
3749                 0x20400, 0x20528,
3750                 0x20540, 0x20614,
3751                 0x21000, 0x21040,
3752                 0x2104c, 0x21060,
3753                 0x210c0, 0x210ec,
3754                 0x21200, 0x21268,
3755                 0x21270, 0x21284,
3756                 0x212fc, 0x21388,
3757                 0x21400, 0x21404,
3758                 0x21500, 0x21518,
3759                 0x2152c, 0x2153c,
3760                 0x21550, 0x21554,
3761                 0x21600, 0x21600,
3762                 0x21608, 0x21628,
3763                 0x21630, 0x2163c,
3764                 0x21700, 0x2171c,
3765                 0x21780, 0x2178c,
3766                 0x21800, 0x21c38,
3767                 0x21c80, 0x21d7c,
3768                 0x21e00, 0x21e04,
3769                 0x22000, 0x2202c,
3770                 0x22100, 0x2213c,
3771                 0x22190, 0x221c8,
3772                 0x22200, 0x22318,
3773                 0x22400, 0x22528,
3774                 0x22540, 0x22614,
3775                 0x23000, 0x23040,
3776                 0x2304c, 0x23060,
3777                 0x230c0, 0x230ec,
3778                 0x23200, 0x23268,
3779                 0x23270, 0x23284,
3780                 0x232fc, 0x23388,
3781                 0x23400, 0x23404,
3782                 0x23500, 0x23518,
3783                 0x2352c, 0x2353c,
3784                 0x23550, 0x23554,
3785                 0x23600, 0x23600,
3786                 0x23608, 0x23628,
3787                 0x23630, 0x2363c,
3788                 0x23700, 0x2371c,
3789                 0x23780, 0x2378c,
3790                 0x23800, 0x23c38,
3791                 0x23c80, 0x23d7c,
3792                 0x23e00, 0x23e04,
3793                 0x24000, 0x2402c,
3794                 0x24100, 0x2413c,
3795                 0x24190, 0x241c8,
3796                 0x24200, 0x24318,
3797                 0x24400, 0x24528,
3798                 0x24540, 0x24614,
3799                 0x25000, 0x25040,
3800                 0x2504c, 0x25060,
3801                 0x250c0, 0x250ec,
3802                 0x25200, 0x25268,
3803                 0x25270, 0x25284,
3804                 0x252fc, 0x25388,
3805                 0x25400, 0x25404,
3806                 0x25500, 0x25518,
3807                 0x2552c, 0x2553c,
3808                 0x25550, 0x25554,
3809                 0x25600, 0x25600,
3810                 0x25608, 0x25628,
3811                 0x25630, 0x2563c,
3812                 0x25700, 0x2571c,
3813                 0x25780, 0x2578c,
3814                 0x25800, 0x25c38,
3815                 0x25c80, 0x25d7c,
3816                 0x25e00, 0x25e04,
3817                 0x26000, 0x2602c,
3818                 0x26100, 0x2613c,
3819                 0x26190, 0x261c8,
3820                 0x26200, 0x26318,
3821                 0x26400, 0x26528,
3822                 0x26540, 0x26614,
3823                 0x27000, 0x27040,
3824                 0x2704c, 0x27060,
3825                 0x270c0, 0x270ec,
3826                 0x27200, 0x27268,
3827                 0x27270, 0x27284,
3828                 0x272fc, 0x27388,
3829                 0x27400, 0x27404,
3830                 0x27500, 0x27518,
3831                 0x2752c, 0x2753c,
3832                 0x27550, 0x27554,
3833                 0x27600, 0x27600,
3834                 0x27608, 0x27628,
3835                 0x27630, 0x2763c,
3836                 0x27700, 0x2771c,
3837                 0x27780, 0x2778c,
3838                 0x27800, 0x27c38,
3839                 0x27c80, 0x27d7c,
3840                 0x27e00, 0x27e04
3841         };
3842         static const unsigned int t5_reg_ranges[] = {
3843                 0x1008, 0x1148,
3844                 0x1180, 0x11b4,
3845                 0x11fc, 0x123c,
3846                 0x1280, 0x173c,
3847                 0x1800, 0x18fc,
3848                 0x3000, 0x3028,
3849                 0x3060, 0x30d8,
3850                 0x30e0, 0x30fc,
3851                 0x3140, 0x357c,
3852                 0x35a8, 0x35cc,
3853                 0x35ec, 0x35ec,
3854                 0x3600, 0x5624,
3855                 0x56cc, 0x575c,
3856                 0x580c, 0x5814,
3857                 0x5890, 0x58bc,
3858                 0x5940, 0x59dc,
3859                 0x59fc, 0x5a18,
3860                 0x5a60, 0x5a9c,
3861                 0x5b94, 0x5bfc,
3862                 0x6000, 0x6040,
3863                 0x6058, 0x614c,
3864                 0x7700, 0x7798,
3865                 0x77c0, 0x78fc,
3866                 0x7b00, 0x7c54,
3867                 0x7d00, 0x7efc,
3868                 0x8dc0, 0x8de0,
3869                 0x8df8, 0x8e84,
3870                 0x8ea0, 0x8f84,
3871                 0x8fc0, 0x90f8,
3872                 0x9400, 0x9470,
3873                 0x9600, 0x96f4,
3874                 0x9800, 0x9808,
3875                 0x9820, 0x983c,
3876                 0x9850, 0x9864,
3877                 0x9c00, 0x9c6c,
3878                 0x9c80, 0x9cec,
3879                 0x9d00, 0x9d6c,
3880                 0x9d80, 0x9dec,
3881                 0x9e00, 0x9e6c,
3882                 0x9e80, 0x9eec,
3883                 0x9f00, 0x9f6c,
3884                 0x9f80, 0xa020,
3885                 0xd004, 0xd03c,
3886                 0xdfc0, 0xdfe0,
3887                 0xe000, 0x11088,
3888                 0x1109c, 0x11110,
3889                 0x11118, 0x1117c,
3890                 0x11190, 0x11204,
3891                 0x19040, 0x1906c,
3892                 0x19078, 0x19080,
3893                 0x1908c, 0x19124,
3894                 0x19150, 0x191b0,
3895                 0x191d0, 0x191e8,
3896                 0x19238, 0x19290,
3897                 0x193f8, 0x19474,
3898                 0x19490, 0x194cc,
3899                 0x194f0, 0x194f8,
3900                 0x19c00, 0x19c60,
3901                 0x19c94, 0x19e10,
3902                 0x19e50, 0x19f34,
3903                 0x19f40, 0x19f50,
3904                 0x19f90, 0x19fe4,
3905                 0x1a000, 0x1a06c,
3906                 0x1a0b0, 0x1a120,
3907                 0x1a128, 0x1a138,
3908                 0x1a190, 0x1a1c4,
3909                 0x1a1fc, 0x1a1fc,
3910                 0x1e008, 0x1e00c,
3911                 0x1e040, 0x1e04c,
3912                 0x1e284, 0x1e290,
3913                 0x1e2c0, 0x1e2c0,
3914                 0x1e2e0, 0x1e2e0,
3915                 0x1e300, 0x1e384,
3916                 0x1e3c0, 0x1e3c8,
3917                 0x1e408, 0x1e40c,
3918                 0x1e440, 0x1e44c,
3919                 0x1e684, 0x1e690,
3920                 0x1e6c0, 0x1e6c0,
3921                 0x1e6e0, 0x1e6e0,
3922                 0x1e700, 0x1e784,
3923                 0x1e7c0, 0x1e7c8,
3924                 0x1e808, 0x1e80c,
3925                 0x1e840, 0x1e84c,
3926                 0x1ea84, 0x1ea90,
3927                 0x1eac0, 0x1eac0,
3928                 0x1eae0, 0x1eae0,
3929                 0x1eb00, 0x1eb84,
3930                 0x1ebc0, 0x1ebc8,
3931                 0x1ec08, 0x1ec0c,
3932                 0x1ec40, 0x1ec4c,
3933                 0x1ee84, 0x1ee90,
3934                 0x1eec0, 0x1eec0,
3935                 0x1eee0, 0x1eee0,
3936                 0x1ef00, 0x1ef84,
3937                 0x1efc0, 0x1efc8,
3938                 0x1f008, 0x1f00c,
3939                 0x1f040, 0x1f04c,
3940                 0x1f284, 0x1f290,
3941                 0x1f2c0, 0x1f2c0,
3942                 0x1f2e0, 0x1f2e0,
3943                 0x1f300, 0x1f384,
3944                 0x1f3c0, 0x1f3c8,
3945                 0x1f408, 0x1f40c,
3946                 0x1f440, 0x1f44c,
3947                 0x1f684, 0x1f690,
3948                 0x1f6c0, 0x1f6c0,
3949                 0x1f6e0, 0x1f6e0,
3950                 0x1f700, 0x1f784,
3951                 0x1f7c0, 0x1f7c8,
3952                 0x1f808, 0x1f80c,
3953                 0x1f840, 0x1f84c,
3954                 0x1fa84, 0x1fa90,
3955                 0x1fac0, 0x1fac0,
3956                 0x1fae0, 0x1fae0,
3957                 0x1fb00, 0x1fb84,
3958                 0x1fbc0, 0x1fbc8,
3959                 0x1fc08, 0x1fc0c,
3960                 0x1fc40, 0x1fc4c,
3961                 0x1fe84, 0x1fe90,
3962                 0x1fec0, 0x1fec0,
3963                 0x1fee0, 0x1fee0,
3964                 0x1ff00, 0x1ff84,
3965                 0x1ffc0, 0x1ffc8,
3966                 0x30000, 0x30030,
3967                 0x30100, 0x30144,
3968                 0x30190, 0x301d0,
3969                 0x30200, 0x30318,
3970                 0x30400, 0x3052c,
3971                 0x30540, 0x3061c,
3972                 0x30800, 0x30834,
3973                 0x308c0, 0x30908,
3974                 0x30910, 0x309ac,
3975                 0x30a00, 0x30a2c,
3976                 0x30a44, 0x30a50,
3977                 0x30a74, 0x30c24,
3978                 0x30d00, 0x30d00,
3979                 0x30d08, 0x30d14,
3980                 0x30d1c, 0x30d20,
3981                 0x30d3c, 0x30d50,
3982                 0x31200, 0x3120c,
3983                 0x31220, 0x31220,
3984                 0x31240, 0x31240,
3985                 0x31600, 0x3160c,
3986                 0x31a00, 0x31a1c,
3987                 0x31e00, 0x31e20,
3988                 0x31e38, 0x31e3c,
3989                 0x31e80, 0x31e80,
3990                 0x31e88, 0x31ea8,
3991                 0x31eb0, 0x31eb4,
3992                 0x31ec8, 0x31ed4,
3993                 0x31fb8, 0x32004,
3994                 0x32200, 0x32200,
3995                 0x32208, 0x32240,
3996                 0x32248, 0x32280,
3997                 0x32288, 0x322c0,
3998                 0x322c8, 0x322fc,
3999                 0x32600, 0x32630,
4000                 0x32a00, 0x32abc,
4001                 0x32b00, 0x32b70,
4002                 0x33000, 0x33048,
4003                 0x33060, 0x3309c,
4004                 0x330f0, 0x33148,
4005                 0x33160, 0x3319c,
4006                 0x331f0, 0x332e4,
4007                 0x332f8, 0x333e4,
4008                 0x333f8, 0x33448,
4009                 0x33460, 0x3349c,
4010                 0x334f0, 0x33548,
4011                 0x33560, 0x3359c,
4012                 0x335f0, 0x336e4,
4013                 0x336f8, 0x337e4,
4014                 0x337f8, 0x337fc,
4015                 0x33814, 0x33814,
4016                 0x3382c, 0x3382c,
4017                 0x33880, 0x3388c,
4018                 0x338e8, 0x338ec,
4019                 0x33900, 0x33948,
4020                 0x33960, 0x3399c,
4021                 0x339f0, 0x33ae4,
4022                 0x33af8, 0x33b10,
4023                 0x33b28, 0x33b28,
4024                 0x33b3c, 0x33b50,
4025                 0x33bf0, 0x33c10,
4026                 0x33c28, 0x33c28,
4027                 0x33c3c, 0x33c50,
4028                 0x33cf0, 0x33cfc,
4029                 0x34000, 0x34030,
4030                 0x34100, 0x34144,
4031                 0x34190, 0x341d0,
4032                 0x34200, 0x34318,
4033                 0x34400, 0x3452c,
4034                 0x34540, 0x3461c,
4035                 0x34800, 0x34834,
4036                 0x348c0, 0x34908,
4037                 0x34910, 0x349ac,
4038                 0x34a00, 0x34a2c,
4039                 0x34a44, 0x34a50,
4040                 0x34a74, 0x34c24,
4041                 0x34d00, 0x34d00,
4042                 0x34d08, 0x34d14,
4043                 0x34d1c, 0x34d20,
4044                 0x34d3c, 0x34d50,
4045                 0x35200, 0x3520c,
4046                 0x35220, 0x35220,
4047                 0x35240, 0x35240,
4048                 0x35600, 0x3560c,
4049                 0x35a00, 0x35a1c,
4050                 0x35e00, 0x35e20,
4051                 0x35e38, 0x35e3c,
4052                 0x35e80, 0x35e80,
4053                 0x35e88, 0x35ea8,
4054                 0x35eb0, 0x35eb4,
4055                 0x35ec8, 0x35ed4,
4056                 0x35fb8, 0x36004,
4057                 0x36200, 0x36200,
4058                 0x36208, 0x36240,
4059                 0x36248, 0x36280,
4060                 0x36288, 0x362c0,
4061                 0x362c8, 0x362fc,
4062                 0x36600, 0x36630,
4063                 0x36a00, 0x36abc,
4064                 0x36b00, 0x36b70,
4065                 0x37000, 0x37048,
4066                 0x37060, 0x3709c,
4067                 0x370f0, 0x37148,
4068                 0x37160, 0x3719c,
4069                 0x371f0, 0x372e4,
4070                 0x372f8, 0x373e4,
4071                 0x373f8, 0x37448,
4072                 0x37460, 0x3749c,
4073                 0x374f0, 0x37548,
4074                 0x37560, 0x3759c,
4075                 0x375f0, 0x376e4,
4076                 0x376f8, 0x377e4,
4077                 0x377f8, 0x377fc,
4078                 0x37814, 0x37814,
4079                 0x3782c, 0x3782c,
4080                 0x37880, 0x3788c,
4081                 0x378e8, 0x378ec,
4082                 0x37900, 0x37948,
4083                 0x37960, 0x3799c,
4084                 0x379f0, 0x37ae4,
4085                 0x37af8, 0x37b10,
4086                 0x37b28, 0x37b28,
4087                 0x37b3c, 0x37b50,
4088                 0x37bf0, 0x37c10,
4089                 0x37c28, 0x37c28,
4090                 0x37c3c, 0x37c50,
4091                 0x37cf0, 0x37cfc,
4092                 0x38000, 0x38030,
4093                 0x38100, 0x38144,
4094                 0x38190, 0x381d0,
4095                 0x38200, 0x38318,
4096                 0x38400, 0x3852c,
4097                 0x38540, 0x3861c,
4098                 0x38800, 0x38834,
4099                 0x388c0, 0x38908,
4100                 0x38910, 0x389ac,
4101                 0x38a00, 0x38a2c,
4102                 0x38a44, 0x38a50,
4103                 0x38a74, 0x38c24,
4104                 0x38d00, 0x38d00,
4105                 0x38d08, 0x38d14,
4106                 0x38d1c, 0x38d20,
4107                 0x38d3c, 0x38d50,
4108                 0x39200, 0x3920c,
4109                 0x39220, 0x39220,
4110                 0x39240, 0x39240,
4111                 0x39600, 0x3960c,
4112                 0x39a00, 0x39a1c,
4113                 0x39e00, 0x39e20,
4114                 0x39e38, 0x39e3c,
4115                 0x39e80, 0x39e80,
4116                 0x39e88, 0x39ea8,
4117                 0x39eb0, 0x39eb4,
4118                 0x39ec8, 0x39ed4,
4119                 0x39fb8, 0x3a004,
4120                 0x3a200, 0x3a200,
4121                 0x3a208, 0x3a240,
4122                 0x3a248, 0x3a280,
4123                 0x3a288, 0x3a2c0,
4124                 0x3a2c8, 0x3a2fc,
4125                 0x3a600, 0x3a630,
4126                 0x3aa00, 0x3aabc,
4127                 0x3ab00, 0x3ab70,
4128                 0x3b000, 0x3b048,
4129                 0x3b060, 0x3b09c,
4130                 0x3b0f0, 0x3b148,
4131                 0x3b160, 0x3b19c,
4132                 0x3b1f0, 0x3b2e4,
4133                 0x3b2f8, 0x3b3e4,
4134                 0x3b3f8, 0x3b448,
4135                 0x3b460, 0x3b49c,
4136                 0x3b4f0, 0x3b548,
4137                 0x3b560, 0x3b59c,
4138                 0x3b5f0, 0x3b6e4,
4139                 0x3b6f8, 0x3b7e4,
4140                 0x3b7f8, 0x3b7fc,
4141                 0x3b814, 0x3b814,
4142                 0x3b82c, 0x3b82c,
4143                 0x3b880, 0x3b88c,
4144                 0x3b8e8, 0x3b8ec,
4145                 0x3b900, 0x3b948,
4146                 0x3b960, 0x3b99c,
4147                 0x3b9f0, 0x3bae4,
4148                 0x3baf8, 0x3bb10,
4149                 0x3bb28, 0x3bb28,
4150                 0x3bb3c, 0x3bb50,
4151                 0x3bbf0, 0x3bc10,
4152                 0x3bc28, 0x3bc28,
4153                 0x3bc3c, 0x3bc50,
4154                 0x3bcf0, 0x3bcfc,
4155                 0x3c000, 0x3c030,
4156                 0x3c100, 0x3c144,
4157                 0x3c190, 0x3c1d0,
4158                 0x3c200, 0x3c318,
4159                 0x3c400, 0x3c52c,
4160                 0x3c540, 0x3c61c,
4161                 0x3c800, 0x3c834,
4162                 0x3c8c0, 0x3c908,
4163                 0x3c910, 0x3c9ac,
4164                 0x3ca00, 0x3ca2c,
4165                 0x3ca44, 0x3ca50,
4166                 0x3ca74, 0x3cc24,
4167                 0x3cd00, 0x3cd00,
4168                 0x3cd08, 0x3cd14,
4169                 0x3cd1c, 0x3cd20,
4170                 0x3cd3c, 0x3cd50,
4171                 0x3d200, 0x3d20c,
4172                 0x3d220, 0x3d220,
4173                 0x3d240, 0x3d240,
4174                 0x3d600, 0x3d60c,
4175                 0x3da00, 0x3da1c,
4176                 0x3de00, 0x3de20,
4177                 0x3de38, 0x3de3c,
4178                 0x3de80, 0x3de80,
4179                 0x3de88, 0x3dea8,
4180                 0x3deb0, 0x3deb4,
4181                 0x3dec8, 0x3ded4,
4182                 0x3dfb8, 0x3e004,
4183                 0x3e200, 0x3e200,
4184                 0x3e208, 0x3e240,
4185                 0x3e248, 0x3e280,
4186                 0x3e288, 0x3e2c0,
4187                 0x3e2c8, 0x3e2fc,
4188                 0x3e600, 0x3e630,
4189                 0x3ea00, 0x3eabc,
4190                 0x3eb00, 0x3eb70,
4191                 0x3f000, 0x3f048,
4192                 0x3f060, 0x3f09c,
4193                 0x3f0f0, 0x3f148,
4194                 0x3f160, 0x3f19c,
4195                 0x3f1f0, 0x3f2e4,
4196                 0x3f2f8, 0x3f3e4,
4197                 0x3f3f8, 0x3f448,
4198                 0x3f460, 0x3f49c,
4199                 0x3f4f0, 0x3f548,
4200                 0x3f560, 0x3f59c,
4201                 0x3f5f0, 0x3f6e4,
4202                 0x3f6f8, 0x3f7e4,
4203                 0x3f7f8, 0x3f7fc,
4204                 0x3f814, 0x3f814,
4205                 0x3f82c, 0x3f82c,
4206                 0x3f880, 0x3f88c,
4207                 0x3f8e8, 0x3f8ec,
4208                 0x3f900, 0x3f948,
4209                 0x3f960, 0x3f99c,
4210                 0x3f9f0, 0x3fae4,
4211                 0x3faf8, 0x3fb10,
4212                 0x3fb28, 0x3fb28,
4213                 0x3fb3c, 0x3fb50,
4214                 0x3fbf0, 0x3fc10,
4215                 0x3fc28, 0x3fc28,
4216                 0x3fc3c, 0x3fc50,
4217                 0x3fcf0, 0x3fcfc,
4218                 0x40000, 0x4000c,
4219                 0x40040, 0x40068,
4220                 0x4007c, 0x40144,
4221                 0x40180, 0x4018c,
4222                 0x40200, 0x40298,
4223                 0x402ac, 0x4033c,
4224                 0x403f8, 0x403fc,
4225                 0x41304, 0x413c4,
4226                 0x41400, 0x4141c,
4227                 0x41480, 0x414d0,
4228                 0x44000, 0x44078,
4229                 0x440c0, 0x44278,
4230                 0x442c0, 0x44478,
4231                 0x444c0, 0x44678,
4232                 0x446c0, 0x44878,
4233                 0x448c0, 0x449fc,
4234                 0x45000, 0x45068,
4235                 0x45080, 0x45084,
4236                 0x450a0, 0x450b0,
4237                 0x45200, 0x45268,
4238                 0x45280, 0x45284,
4239                 0x452a0, 0x452b0,
4240                 0x460c0, 0x460e4,
4241                 0x47000, 0x4708c,
4242                 0x47200, 0x47250,
4243                 0x47400, 0x47420,
4244                 0x47600, 0x47618,
4245                 0x47800, 0x47814,
4246                 0x48000, 0x4800c,
4247                 0x48040, 0x48068,
4248                 0x4807c, 0x48144,
4249                 0x48180, 0x4818c,
4250                 0x48200, 0x48298,
4251                 0x482ac, 0x4833c,
4252                 0x483f8, 0x483fc,
4253                 0x49304, 0x493c4,
4254                 0x49400, 0x4941c,
4255                 0x49480, 0x494d0,
4256                 0x4c000, 0x4c078,
4257                 0x4c0c0, 0x4c278,
4258                 0x4c2c0, 0x4c478,
4259                 0x4c4c0, 0x4c678,
4260                 0x4c6c0, 0x4c878,
4261                 0x4c8c0, 0x4c9fc,
4262                 0x4d000, 0x4d068,
4263                 0x4d080, 0x4d084,
4264                 0x4d0a0, 0x4d0b0,
4265                 0x4d200, 0x4d268,
4266                 0x4d280, 0x4d284,
4267                 0x4d2a0, 0x4d2b0,
4268                 0x4e0c0, 0x4e0e4,
4269                 0x4f000, 0x4f08c,
4270                 0x4f200, 0x4f250,
4271                 0x4f400, 0x4f420,
4272                 0x4f600, 0x4f618,
4273                 0x4f800, 0x4f814,
4274                 0x50000, 0x500cc,
4275                 0x50400, 0x50400,
4276                 0x50800, 0x508cc,
4277                 0x50c00, 0x50c00,
4278                 0x51000, 0x5101c,
4279                 0x51300, 0x51308,
4280         };
4281
4282         if (is_t4(sc)) {
4283                 reg_ranges = &t4_reg_ranges[0];
4284                 n = nitems(t4_reg_ranges);
4285         } else {
4286                 reg_ranges = &t5_reg_ranges[0];
4287                 n = nitems(t5_reg_ranges);
4288         }
4289
4290         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4291         for (i = 0; i < n; i += 2)
4292                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4293 }
4294
4295 static void
4296 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4297 {
4298         struct ifnet *ifp = pi->ifp;
4299         struct sge_txq *txq;
4300         int i, drops;
4301         struct port_stats *s = &pi->stats;
4302         struct timeval tv;
4303         const struct timeval interval = {0, 250000};    /* 250ms */
4304
4305         getmicrotime(&tv);
4306         timevalsub(&tv, &interval);
4307         if (timevalcmp(&tv, &pi->last_refreshed, <))
4308                 return;
4309
4310         t4_get_port_stats(sc, pi->tx_chan, s);
4311
4312         ifp->if_opackets = s->tx_frames - s->tx_pause;
4313         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4314         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4315         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4316         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4317         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4318         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4319             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4320             s->rx_trunc3;
4321         for (i = 0; i < NCHAN; i++) {
4322                 if (pi->rx_chan_map & (1 << i)) {
4323                         uint32_t v;
4324
4325                         mtx_lock(&sc->regwin_lock);
4326                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4327                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4328                         mtx_unlock(&sc->regwin_lock);
4329                         ifp->if_iqdrops += v;
4330                 }
4331         }
4332
4333         drops = s->tx_drop;
4334         for_each_txq(pi, i, txq)
4335                 drops += counter_u64_fetch(txq->r->drops);
4336         ifp->if_snd.ifq_drops = drops;
4337
4338         ifp->if_oerrors = s->tx_error_frames;
4339         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4340             s->rx_fcs_err + s->rx_len_err;
4341
4342         getmicrotime(&pi->last_refreshed);
4343 }
4344
4345 static void
4346 cxgbe_tick(void *arg)
4347 {
4348         struct port_info *pi = arg;
4349         struct adapter *sc = pi->adapter;
4350         struct ifnet *ifp = pi->ifp;
4351
4352         PORT_LOCK(pi);
4353         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4354                 PORT_UNLOCK(pi);
4355                 return; /* without scheduling another callout */
4356         }
4357
4358         cxgbe_refresh_stats(sc, pi);
4359
4360         callout_schedule(&pi->tick, hz);
4361         PORT_UNLOCK(pi);
4362 }
4363
4364 static void
4365 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4366 {
4367         struct ifnet *vlan;
4368
4369         if (arg != ifp || ifp->if_type != IFT_ETHER)
4370                 return;
4371
4372         vlan = VLAN_DEVAT(ifp, vid);
4373         VLAN_SETCOOKIE(vlan, ifp);
4374 }
4375
4376 static int
4377 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4378 {
4379
4380 #ifdef INVARIANTS
4381         panic("%s: opcode 0x%02x on iq %p with payload %p",
4382             __func__, rss->opcode, iq, m);
4383 #else
4384         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4385             __func__, rss->opcode, iq, m);
4386         m_freem(m);
4387 #endif
4388         return (EDOOFUS);
4389 }
4390
4391 int
4392 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4393 {
4394         uintptr_t *loc, new;
4395
4396         if (opcode >= nitems(sc->cpl_handler))
4397                 return (EINVAL);
4398
4399         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4400         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4401         atomic_store_rel_ptr(loc, new);
4402
4403         return (0);
4404 }
4405
4406 static int
4407 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4408 {
4409
4410 #ifdef INVARIANTS
4411         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4412 #else
4413         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4414             __func__, iq, ctrl);
4415 #endif
4416         return (EDOOFUS);
4417 }
4418
4419 int
4420 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4421 {
4422         uintptr_t *loc, new;
4423
4424         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4425         loc = (uintptr_t *) &sc->an_handler;
4426         atomic_store_rel_ptr(loc, new);
4427
4428         return (0);
4429 }
4430
4431 static int
4432 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4433 {
4434         const struct cpl_fw6_msg *cpl =
4435             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4436
4437 #ifdef INVARIANTS
4438         panic("%s: fw_msg type %d", __func__, cpl->type);
4439 #else
4440         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4441 #endif
4442         return (EDOOFUS);
4443 }
4444
4445 int
4446 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4447 {
4448         uintptr_t *loc, new;
4449
4450         if (type >= nitems(sc->fw_msg_handler))
4451                 return (EINVAL);
4452
4453         /*
4454          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4455          * handler dispatch table.  Reject any attempt to install a handler for
4456          * this subtype.
4457          */
4458         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4459                 return (EINVAL);
4460
4461         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4462         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4463         atomic_store_rel_ptr(loc, new);
4464
4465         return (0);
4466 }
4467
4468 static int
4469 t4_sysctls(struct adapter *sc)
4470 {
4471         struct sysctl_ctx_list *ctx;
4472         struct sysctl_oid *oid;
4473         struct sysctl_oid_list *children, *c0;
4474         static char *caps[] = {
4475                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4476                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4477                     "\6HASHFILTER\7ETHOFLD",
4478                 "\20\1TOE",                             /* caps[2] toecaps */
4479                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4480                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4481                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4482                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4483                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4484                     "\4PO_INITIAOR\5PO_TARGET"
4485         };
4486         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4487
4488         ctx = device_get_sysctl_ctx(sc->dev);
4489
4490         /*
4491          * dev.t4nex.X.
4492          */
4493         oid = device_get_sysctl_tree(sc->dev);
4494         c0 = children = SYSCTL_CHILDREN(oid);
4495
4496         sc->sc_do_rxcopy = 1;
4497         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4498             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4499
4500         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4501             sc->params.nports, "# of ports");
4502
4503         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4504             NULL, chip_rev(sc), "chip hardware revision");
4505
4506         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4507             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4508
4509         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4510             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4511
4512         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4513             sc->cfcsum, "config file checksum");
4514
4515         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4516             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4517             sysctl_bitfield, "A", "available doorbells");
4518
4519         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4520             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4521             sysctl_bitfield, "A", "available link capabilities");
4522
4523         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4524             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4525             sysctl_bitfield, "A", "available NIC capabilities");
4526
4527         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4528             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4529             sysctl_bitfield, "A", "available TCP offload capabilities");
4530
4531         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4532             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4533             sysctl_bitfield, "A", "available RDMA capabilities");
4534
4535         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4536             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4537             sysctl_bitfield, "A", "available iSCSI capabilities");
4538
4539         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4540             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4541             sysctl_bitfield, "A", "available FCoE capabilities");
4542
4543         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4544             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4545
4546         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4547             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4548             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4549             "interrupt holdoff timer values (us)");
4550
4551         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4552             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4553             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4554             "interrupt holdoff packet counter values");
4555
4556         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4557             NULL, sc->tids.nftids, "number of filters");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4560             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4561             "chip temperature (in Celsius)");
4562
4563         t4_sge_sysctls(sc, ctx, children);
4564
4565         sc->lro_timeout = 100;
4566         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4567             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4568
4569         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4570             &sc->debug_flags, 0, "flags to enable runtime debugging");
4571
4572 #ifdef SBUF_DRAIN
4573         /*
4574          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4575          */
4576         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4577             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4578             "logs and miscellaneous information");
4579         children = SYSCTL_CHILDREN(oid);
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4583             sysctl_cctrl, "A", "congestion control");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4586             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4587             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4590             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4591             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4592
4593         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4594             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4595             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4596
4597         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4598             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4599             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4600
4601         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4602             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4603             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4604
4605         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4606             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4607             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4608
4609         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4610             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4611             sysctl_cim_la, "A", "CIM logic analyzer");
4612
4613         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4614             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4615             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4616
4617         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4618             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4619             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4620
4621         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4622             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4623             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4624
4625         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4626             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4627             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4628
4629         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4630             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4631             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4632
4633         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4634             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4635             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4636
4637         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4638             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4639             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4640
4641         if (is_t5(sc)) {
4642                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4643                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4644                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4645
4646                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4647                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4648                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4649         }
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4653             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4657             sysctl_cim_qcfg, "A", "CIM queue configuration");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4661             sysctl_cpl_stats, "A", "CPL statistics");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4665             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4669             sysctl_devlog, "A", "firmware's device log");
4670
4671         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4672             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4673             sysctl_fcoe_stats, "A", "FCoE statistics");
4674
4675         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4676             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4677             sysctl_hw_sched, "A", "hardware scheduler ");
4678
4679         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4680             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4681             sysctl_l2t, "A", "hardware L2 table");
4682
4683         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4684             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4685             sysctl_lb_stats, "A", "loopback statistics");
4686
4687         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4688             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4689             sysctl_meminfo, "A", "memory regions");
4690
4691         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4692             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4693             sysctl_mps_tcam, "A", "MPS TCAM entries");
4694
4695         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4696             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4697             sysctl_path_mtus, "A", "path MTUs");
4698
4699         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4700             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4701             sysctl_pm_stats, "A", "PM statistics");
4702
4703         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4704             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4705             sysctl_rdma_stats, "A", "RDMA statistics");
4706
4707         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4708             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4709             sysctl_tcp_stats, "A", "TCP statistics");
4710
4711         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4712             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4713             sysctl_tids, "A", "TID information");
4714
4715         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4716             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4717             sysctl_tp_err_stats, "A", "TP error statistics");
4718
4719         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4720             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4721             sysctl_tp_la, "A", "TP logic analyzer");
4722
4723         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4724             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4725             sysctl_tx_rate, "A", "Tx rate");
4726
4727         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4728             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4729             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4730
4731         if (is_t5(sc)) {
4732                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4733                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4734                     sysctl_wcwr_stats, "A", "write combined work requests");
4735         }
4736 #endif
4737
4738 #ifdef TCP_OFFLOAD
4739         if (is_offload(sc)) {
4740                 /*
4741                  * dev.t4nex.X.toe.
4742                  */
4743                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4744                     NULL, "TOE parameters");
4745                 children = SYSCTL_CHILDREN(oid);
4746
4747                 sc->tt.sndbuf = 256 * 1024;
4748                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4749                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4750
4751                 sc->tt.ddp = 0;
4752                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4753                     &sc->tt.ddp, 0, "DDP allowed");
4754
4755                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4756                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4757                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4758
4759                 sc->tt.ddp_thres =
4760                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4761                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4762                     &sc->tt.ddp_thres, 0, "DDP threshold");
4763
4764                 sc->tt.rx_coalesce = 1;
4765                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4766                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4767
4768                 sc->tt.tx_align = 1;
4769                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4770                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4771         }
4772 #endif
4773
4774
4775         return (0);
4776 }
4777
4778 static int
4779 cxgbe_sysctls(struct port_info *pi)
4780 {
4781         struct sysctl_ctx_list *ctx;
4782         struct sysctl_oid *oid;
4783         struct sysctl_oid_list *children;
4784         struct adapter *sc = pi->adapter;
4785
4786         ctx = device_get_sysctl_ctx(pi->dev);
4787
4788         /*
4789          * dev.cxgbe.X.
4790          */
4791         oid = device_get_sysctl_tree(pi->dev);
4792         children = SYSCTL_CHILDREN(oid);
4793
4794         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4795            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4796         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4797                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4798                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4799                     "PHY temperature (in Celsius)");
4800                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4801                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4802                     "PHY firmware version");
4803         }
4804         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4805             &pi->nrxq, 0, "# of rx queues");
4806         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4807             &pi->ntxq, 0, "# of tx queues");
4808         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4809             &pi->first_rxq, 0, "index of first rx queue");
4810         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4811             &pi->first_txq, 0, "index of first tx queue");
4812         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4813             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4814             "Reserve queue 0 for non-flowid packets");
4815
4816 #ifdef TCP_OFFLOAD
4817         if (is_offload(sc)) {
4818                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4819                     &pi->nofldrxq, 0,
4820                     "# of rx queues for offloaded TCP connections");
4821                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4822                     &pi->nofldtxq, 0,
4823                     "# of tx queues for offloaded TCP connections");
4824                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4825                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4826                     "index of first TOE rx queue");
4827                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4828                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4829                     "index of first TOE tx queue");
4830         }
4831 #endif
4832 #ifdef DEV_NETMAP
4833         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4834             &pi->nnmrxq, 0, "# of rx queues for netmap");
4835         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4836             &pi->nnmtxq, 0, "# of tx queues for netmap");
4837         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4838             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4839             "index of first netmap rx queue");
4840         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4841             CTLFLAG_RD, &pi->first_nm_txq, 0,
4842             "index of first netmap tx queue");
4843 #endif
4844
4845         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4846             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4847             "holdoff timer index");
4848         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4849             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4850             "holdoff packet counter index");
4851
4852         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4853             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4854             "rx queue size");
4855         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4856             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4857             "tx queue size");
4858
4859         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4860             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4861             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4862
4863         /*
4864          * dev.cxgbe.X.stats.
4865          */
4866         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4867             NULL, "port statistics");
4868         children = SYSCTL_CHILDREN(oid);
4869         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4870             &pi->tx_parse_error, 0,
4871             "# of tx packets with invalid length or # of segments");
4872
4873 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4874         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4875             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4876             sysctl_handle_t4_reg64, "QU", desc)
4877
4878         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4880         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4882         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4884         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4886         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4887             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4888         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4890         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4891             "# of tx frames in this range",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4893         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4894             "# of tx frames in this range",
4895             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4896         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4897             "# of tx frames in this range",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4899         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4900             "# of tx frames in this range",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4902         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4903             "# of tx frames in this range",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4905         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4906             "# of tx frames in this range",
4907             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4908         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4909             "# of tx frames in this range",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4911         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4913         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4927         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4928             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4929         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4931
4932         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4934         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4936         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4938         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4940         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4942         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4944         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4946         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4947             "# of frames received with bad FCS",
4948             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4949         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4950             "# of frames received with length error",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4952         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4954         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4956         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4957             "# of rx frames in this range",
4958             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4959         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4960             "# of rx frames in this range",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4962         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4963             "# of rx frames in this range",
4964             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4965         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4966             "# of rx frames in this range",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4968         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4969             "# of rx frames in this range",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4972             "# of rx frames in this range",
4973             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4974         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4975             "# of rx frames in this range",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4982             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4983         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4992             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4993         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4994             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4995
4996 #undef SYSCTL_ADD_T4_REG64
4997
4998 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4999         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5000             &pi->stats.name, desc)
5001
5002         /* We get these from port_stats and they may be stale by upto 1s */
5003         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5004             "# drops due to buffer-group 0 overflows");
5005         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5006             "# drops due to buffer-group 1 overflows");
5007         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5008             "# drops due to buffer-group 2 overflows");
5009         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5010             "# drops due to buffer-group 3 overflows");
5011         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5012             "# of buffer-group 0 truncated packets");
5013         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5014             "# of buffer-group 1 truncated packets");
5015         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5016             "# of buffer-group 2 truncated packets");
5017         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5018             "# of buffer-group 3 truncated packets");
5019
5020 #undef SYSCTL_ADD_T4_PORTSTAT
5021
5022         return (0);
5023 }
5024
5025 static int
5026 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5027 {
5028         int rc, *i, space = 0;
5029         struct sbuf sb;
5030
5031         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5032         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5033                 if (space)
5034                         sbuf_printf(&sb, " ");
5035                 sbuf_printf(&sb, "%d", *i);
5036                 space = 1;
5037         }
5038         sbuf_finish(&sb);
5039         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5040         sbuf_delete(&sb);
5041         return (rc);
5042 }
5043
5044 static int
5045 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5046 {
5047         int rc;
5048         struct sbuf *sb;
5049
5050         rc = sysctl_wire_old_buffer(req, 0);
5051         if (rc != 0)
5052                 return(rc);
5053
5054         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5055         if (sb == NULL)
5056                 return (ENOMEM);
5057
5058         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5059         rc = sbuf_finish(sb);
5060         sbuf_delete(sb);
5061
5062         return (rc);
5063 }
5064
5065 static int
5066 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5067 {
5068         struct port_info *pi = arg1;
5069         int op = arg2;
5070         struct adapter *sc = pi->adapter;
5071         u_int v;
5072         int rc;
5073
5074         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5075         if (rc)
5076                 return (rc);
5077         /* XXX: magic numbers */
5078         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5079             &v);
5080         end_synchronized_op(sc, 0);
5081         if (rc)
5082                 return (rc);
5083         if (op == 0)
5084                 v /= 256;
5085
5086         rc = sysctl_handle_int(oidp, &v, 0, req);
5087         return (rc);
5088 }
5089
5090 static int
5091 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5092 {
5093         struct port_info *pi = arg1;
5094         int rc, val;
5095
5096         val = pi->rsrv_noflowq;
5097         rc = sysctl_handle_int(oidp, &val, 0, req);
5098         if (rc != 0 || req->newptr == NULL)
5099                 return (rc);
5100
5101         if ((val >= 1) && (pi->ntxq > 1))
5102                 pi->rsrv_noflowq = 1;
5103         else
5104                 pi->rsrv_noflowq = 0;
5105
5106         return (rc);
5107 }
5108
5109 static int
5110 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5111 {
5112         struct port_info *pi = arg1;
5113         struct adapter *sc = pi->adapter;
5114         int idx, rc, i;
5115         struct sge_rxq *rxq;
5116 #ifdef TCP_OFFLOAD
5117         struct sge_ofld_rxq *ofld_rxq;
5118 #endif
5119         uint8_t v;
5120
5121         idx = pi->tmr_idx;
5122
5123         rc = sysctl_handle_int(oidp, &idx, 0, req);
5124         if (rc != 0 || req->newptr == NULL)
5125                 return (rc);
5126
5127         if (idx < 0 || idx >= SGE_NTIMERS)
5128                 return (EINVAL);
5129
5130         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5131             "t4tmr");
5132         if (rc)
5133                 return (rc);
5134
5135         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5136         for_each_rxq(pi, i, rxq) {
5137 #ifdef atomic_store_rel_8
5138                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5139 #else
5140                 rxq->iq.intr_params = v;
5141 #endif
5142         }
5143 #ifdef TCP_OFFLOAD
5144         for_each_ofld_rxq(pi, i, ofld_rxq) {
5145 #ifdef atomic_store_rel_8
5146                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5147 #else
5148                 ofld_rxq->iq.intr_params = v;
5149 #endif
5150         }
5151 #endif
5152         pi->tmr_idx = idx;
5153
5154         end_synchronized_op(sc, LOCK_HELD);
5155         return (0);
5156 }
5157
5158 static int
5159 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5160 {
5161         struct port_info *pi = arg1;
5162         struct adapter *sc = pi->adapter;
5163         int idx, rc;
5164
5165         idx = pi->pktc_idx;
5166
5167         rc = sysctl_handle_int(oidp, &idx, 0, req);
5168         if (rc != 0 || req->newptr == NULL)
5169                 return (rc);
5170
5171         if (idx < -1 || idx >= SGE_NCOUNTERS)
5172                 return (EINVAL);
5173
5174         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5175             "t4pktc");
5176         if (rc)
5177                 return (rc);
5178
5179         if (pi->flags & PORT_INIT_DONE)
5180                 rc = EBUSY; /* cannot be changed once the queues are created */
5181         else
5182                 pi->pktc_idx = idx;
5183
5184         end_synchronized_op(sc, LOCK_HELD);
5185         return (rc);
5186 }
5187
5188 static int
5189 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5190 {
5191         struct port_info *pi = arg1;
5192         struct adapter *sc = pi->adapter;
5193         int qsize, rc;
5194
5195         qsize = pi->qsize_rxq;
5196
5197         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5198         if (rc != 0 || req->newptr == NULL)
5199                 return (rc);
5200
5201         if (qsize < 128 || (qsize & 7))
5202                 return (EINVAL);
5203
5204         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5205             "t4rxqs");
5206         if (rc)
5207                 return (rc);
5208
5209         if (pi->flags & PORT_INIT_DONE)
5210                 rc = EBUSY; /* cannot be changed once the queues are created */
5211         else
5212                 pi->qsize_rxq = qsize;
5213
5214         end_synchronized_op(sc, LOCK_HELD);
5215         return (rc);
5216 }
5217
5218 static int
5219 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5220 {
5221         struct port_info *pi = arg1;
5222         struct adapter *sc = pi->adapter;
5223         int qsize, rc;
5224
5225         qsize = pi->qsize_txq;
5226
5227         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5228         if (rc != 0 || req->newptr == NULL)
5229                 return (rc);
5230
5231         if (qsize < 128 || qsize > 65536)
5232                 return (EINVAL);
5233
5234         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5235             "t4txqs");
5236         if (rc)
5237                 return (rc);
5238
5239         if (pi->flags & PORT_INIT_DONE)
5240                 rc = EBUSY; /* cannot be changed once the queues are created */
5241         else
5242                 pi->qsize_txq = qsize;
5243
5244         end_synchronized_op(sc, LOCK_HELD);
5245         return (rc);
5246 }
5247
5248 static int
5249 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5250 {
5251         struct port_info *pi = arg1;
5252         struct adapter *sc = pi->adapter;
5253         struct link_config *lc = &pi->link_cfg;
5254         int rc;
5255
5256         if (req->newptr == NULL) {
5257                 struct sbuf *sb;
5258                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5259
5260                 rc = sysctl_wire_old_buffer(req, 0);
5261                 if (rc != 0)
5262                         return(rc);
5263
5264                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5265                 if (sb == NULL)
5266                         return (ENOMEM);
5267
5268                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5269                 rc = sbuf_finish(sb);
5270                 sbuf_delete(sb);
5271         } else {
5272                 char s[2];
5273                 int n;
5274
5275                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5276                 s[1] = 0;
5277
5278                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5279                 if (rc != 0)
5280                         return(rc);
5281
5282                 if (s[1] != 0)
5283                         return (EINVAL);
5284                 if (s[0] < '0' || s[0] > '9')
5285                         return (EINVAL);        /* not a number */
5286                 n = s[0] - '0';
5287                 if (n & ~(PAUSE_TX | PAUSE_RX))
5288                         return (EINVAL);        /* some other bit is set too */
5289
5290                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5291                 if (rc)
5292                         return (rc);
5293                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5294                         int link_ok = lc->link_ok;
5295
5296                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5297                         lc->requested_fc |= n;
5298                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5299                         lc->link_ok = link_ok;  /* restore */
5300                 }
5301                 end_synchronized_op(sc, 0);
5302         }
5303
5304         return (rc);
5305 }
5306
5307 static int
5308 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5309 {
5310         struct adapter *sc = arg1;
5311         int reg = arg2;
5312         uint64_t val;
5313
5314         val = t4_read_reg64(sc, reg);
5315
5316         return (sysctl_handle_64(oidp, &val, 0, req));
5317 }
5318
5319 static int
5320 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5321 {
5322         struct adapter *sc = arg1;
5323         int rc, t;
5324         uint32_t param, val;
5325
5326         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5327         if (rc)
5328                 return (rc);
5329         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5330             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5331             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5332         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5333         end_synchronized_op(sc, 0);
5334         if (rc)
5335                 return (rc);
5336
5337         /* unknown is returned as 0 but we display -1 in that case */
5338         t = val == 0 ? -1 : val;
5339
5340         rc = sysctl_handle_int(oidp, &t, 0, req);
5341         return (rc);
5342 }
5343
5344 #ifdef SBUF_DRAIN
5345 static int
5346 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5347 {
5348         struct adapter *sc = arg1;
5349         struct sbuf *sb;
5350         int rc, i;
5351         uint16_t incr[NMTUS][NCCTRL_WIN];
5352         static const char *dec_fac[] = {
5353                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5354                 "0.9375"
5355         };
5356
5357         rc = sysctl_wire_old_buffer(req, 0);
5358         if (rc != 0)
5359                 return (rc);
5360
5361         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5362         if (sb == NULL)
5363                 return (ENOMEM);
5364
5365         t4_read_cong_tbl(sc, incr);
5366
5367         for (i = 0; i < NCCTRL_WIN; ++i) {
5368                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5369                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5370                     incr[5][i], incr[6][i], incr[7][i]);
5371                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5372                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5373                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5374                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5375         }
5376
5377         rc = sbuf_finish(sb);
5378         sbuf_delete(sb);
5379
5380         return (rc);
5381 }
5382
5383 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5384         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5385         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5386         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5387 };
5388
5389 static int
5390 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5391 {
5392         struct adapter *sc = arg1;
5393         struct sbuf *sb;
5394         int rc, i, n, qid = arg2;
5395         uint32_t *buf, *p;
5396         char *qtype;
5397         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5398
5399         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5400             ("%s: bad qid %d\n", __func__, qid));
5401
5402         if (qid < CIM_NUM_IBQ) {
5403                 /* inbound queue */
5404                 qtype = "IBQ";
5405                 n = 4 * CIM_IBQ_SIZE;
5406                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5407                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5408         } else {
5409                 /* outbound queue */
5410                 qtype = "OBQ";
5411                 qid -= CIM_NUM_IBQ;
5412                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5413                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5414                 rc = t4_read_cim_obq(sc, qid, buf, n);
5415         }
5416
5417         if (rc < 0) {
5418                 rc = -rc;
5419                 goto done;
5420         }
5421         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5422
5423         rc = sysctl_wire_old_buffer(req, 0);
5424         if (rc != 0)
5425                 goto done;
5426
5427         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5428         if (sb == NULL) {
5429                 rc = ENOMEM;
5430                 goto done;
5431         }
5432
5433         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5434         for (i = 0, p = buf; i < n; i += 16, p += 4)
5435                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5436                     p[2], p[3]);
5437
5438         rc = sbuf_finish(sb);
5439         sbuf_delete(sb);
5440 done:
5441         free(buf, M_CXGBE);
5442         return (rc);
5443 }
5444
5445 static int
5446 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5447 {
5448         struct adapter *sc = arg1;
5449         u_int cfg;
5450         struct sbuf *sb;
5451         uint32_t *buf, *p;
5452         int rc;
5453
5454         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5455         if (rc != 0)
5456                 return (rc);
5457
5458         rc = sysctl_wire_old_buffer(req, 0);
5459         if (rc != 0)
5460                 return (rc);
5461
5462         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5463         if (sb == NULL)
5464                 return (ENOMEM);
5465
5466         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5467             M_ZERO | M_WAITOK);
5468
5469         rc = -t4_cim_read_la(sc, buf, NULL);
5470         if (rc != 0)
5471                 goto done;
5472
5473         sbuf_printf(sb, "Status   Data      PC%s",
5474             cfg & F_UPDBGLACAPTPCONLY ? "" :
5475             "     LS0Stat  LS0Addr             LS0Data");
5476
5477         KASSERT((sc->params.cim_la_size & 7) == 0,
5478             ("%s: p will walk off the end of buf", __func__));
5479
5480         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5481                 if (cfg & F_UPDBGLACAPTPCONLY) {
5482                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5483                             p[6], p[7]);
5484                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5485                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5486                             p[4] & 0xff, p[5] >> 8);
5487                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5488                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5489                             p[1] & 0xf, p[2] >> 4);
5490                 } else {
5491                         sbuf_printf(sb,
5492                             "\n  %02x   %x%07x %x%07x %08x %08x "
5493                             "%08x%08x%08x%08x",
5494                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5495                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5496                             p[6], p[7]);
5497                 }
5498         }
5499
5500         rc = sbuf_finish(sb);
5501         sbuf_delete(sb);
5502 done:
5503         free(buf, M_CXGBE);
5504         return (rc);
5505 }
5506
5507 static int
5508 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5509 {
5510         struct adapter *sc = arg1;
5511         u_int i;
5512         struct sbuf *sb;
5513         uint32_t *buf, *p;
5514         int rc;
5515
5516         rc = sysctl_wire_old_buffer(req, 0);
5517         if (rc != 0)
5518                 return (rc);
5519
5520         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5521         if (sb == NULL)
5522                 return (ENOMEM);
5523
5524         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5525             M_ZERO | M_WAITOK);
5526
5527         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5528         p = buf;
5529
5530         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5531                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5532                     p[1], p[0]);
5533         }
5534
5535         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5536         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5537                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5538                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5539                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5540                     (p[1] >> 2) | ((p[2] & 3) << 30),
5541                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5542                     p[0] & 1);
5543         }
5544
5545         rc = sbuf_finish(sb);
5546         sbuf_delete(sb);
5547         free(buf, M_CXGBE);
5548         return (rc);
5549 }
5550
5551 static int
5552 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5553 {
5554         struct adapter *sc = arg1;
5555         u_int i;
5556         struct sbuf *sb;
5557         uint32_t *buf, *p;
5558         int rc;
5559
5560         rc = sysctl_wire_old_buffer(req, 0);
5561         if (rc != 0)
5562                 return (rc);
5563
5564         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5565         if (sb == NULL)
5566                 return (ENOMEM);
5567
5568         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5569             M_ZERO | M_WAITOK);
5570
5571         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5572         p = buf;
5573
5574         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5575         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5576                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5577                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5578                     p[4], p[3], p[2], p[1], p[0]);
5579         }
5580
5581         sbuf_printf(sb, "\n\nCntl ID               Data");
5582         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5583                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5584                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5585         }
5586
5587         rc = sbuf_finish(sb);
5588         sbuf_delete(sb);
5589         free(buf, M_CXGBE);
5590         return (rc);
5591 }
5592
5593 static int
5594 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5595 {
5596         struct adapter *sc = arg1;
5597         struct sbuf *sb;
5598         int rc, i;
5599         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5600         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5601         uint16_t thres[CIM_NUM_IBQ];
5602         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5603         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5604         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5605
5606         if (is_t4(sc)) {
5607                 cim_num_obq = CIM_NUM_OBQ;
5608                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5609                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5610         } else {
5611                 cim_num_obq = CIM_NUM_OBQ_T5;
5612                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5613                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5614         }
5615         nq = CIM_NUM_IBQ + cim_num_obq;
5616
5617         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5618         if (rc == 0)
5619                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5620         if (rc != 0)
5621                 return (rc);
5622
5623         t4_read_cimq_cfg(sc, base, size, thres);
5624
5625         rc = sysctl_wire_old_buffer(req, 0);
5626         if (rc != 0)
5627                 return (rc);
5628
5629         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5630         if (sb == NULL)
5631                 return (ENOMEM);
5632
5633         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5634
5635         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5636                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5637                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5638                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5639                     G_QUEREMFLITS(p[2]) * 16);
5640         for ( ; i < nq; i++, p += 4, wr += 2)
5641                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5642                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5643                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5644                     G_QUEREMFLITS(p[2]) * 16);
5645
5646         rc = sbuf_finish(sb);
5647         sbuf_delete(sb);
5648
5649         return (rc);
5650 }
5651
5652 static int
5653 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5654 {
5655         struct adapter *sc = arg1;
5656         struct sbuf *sb;
5657         int rc;
5658         struct tp_cpl_stats stats;
5659
5660         rc = sysctl_wire_old_buffer(req, 0);
5661         if (rc != 0)
5662                 return (rc);
5663
5664         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5665         if (sb == NULL)
5666                 return (ENOMEM);
5667
5668         t4_tp_get_cpl_stats(sc, &stats);
5669
5670         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5671             "channel 3\n");
5672         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5673                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5674         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5675                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5676
5677         rc = sbuf_finish(sb);
5678         sbuf_delete(sb);
5679
5680         return (rc);
5681 }
5682
5683 static int
5684 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5685 {
5686         struct adapter *sc = arg1;
5687         struct sbuf *sb;
5688         int rc;
5689         struct tp_usm_stats stats;
5690
5691         rc = sysctl_wire_old_buffer(req, 0);
5692         if (rc != 0)
5693                 return(rc);
5694
5695         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5696         if (sb == NULL)
5697                 return (ENOMEM);
5698
5699         t4_get_usm_stats(sc, &stats);
5700
5701         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5702         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5703         sbuf_printf(sb, "Drops:  %u", stats.drops);
5704
5705         rc = sbuf_finish(sb);
5706         sbuf_delete(sb);
5707
5708         return (rc);
5709 }
5710
5711 const char *devlog_level_strings[] = {
5712         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5713         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5714         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5715         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5716         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5717         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5718 };
5719
5720 const char *devlog_facility_strings[] = {
5721         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5722         [FW_DEVLOG_FACILITY_CF]         = "CF",
5723         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5724         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5725         [FW_DEVLOG_FACILITY_RES]        = "RES",
5726         [FW_DEVLOG_FACILITY_HW]         = "HW",
5727         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5728         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5729         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5730         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5731         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5732         [FW_DEVLOG_FACILITY_VI]         = "VI",
5733         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5734         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5735         [FW_DEVLOG_FACILITY_TM]         = "TM",
5736         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5737         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5738         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5739         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5740         [FW_DEVLOG_FACILITY_RI]         = "RI",
5741         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5742         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5743         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5744         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5745 };
5746
5747 static int
5748 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5749 {
5750         struct adapter *sc = arg1;
5751         struct devlog_params *dparams = &sc->params.devlog;
5752         struct fw_devlog_e *buf, *e;
5753         int i, j, rc, nentries, first = 0, m;
5754         struct sbuf *sb;
5755         uint64_t ftstamp = UINT64_MAX;
5756
5757         if (dparams->start == 0) {
5758                 dparams->memtype = FW_MEMTYPE_EDC0;
5759                 dparams->start = 0x84000;
5760                 dparams->size = 32768;
5761         }
5762
5763         nentries = dparams->size / sizeof(struct fw_devlog_e);
5764
5765         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5766         if (buf == NULL)
5767                 return (ENOMEM);
5768
5769         m = fwmtype_to_hwmtype(dparams->memtype);
5770         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5771         if (rc != 0)
5772                 goto done;
5773
5774         for (i = 0; i < nentries; i++) {
5775                 e = &buf[i];
5776
5777                 if (e->timestamp == 0)
5778                         break;  /* end */
5779
5780                 e->timestamp = be64toh(e->timestamp);
5781                 e->seqno = be32toh(e->seqno);
5782                 for (j = 0; j < 8; j++)
5783                         e->params[j] = be32toh(e->params[j]);
5784
5785                 if (e->timestamp < ftstamp) {
5786                         ftstamp = e->timestamp;
5787                         first = i;
5788                 }
5789         }
5790
5791         if (buf[first].timestamp == 0)
5792                 goto done;      /* nothing in the log */
5793
5794         rc = sysctl_wire_old_buffer(req, 0);
5795         if (rc != 0)
5796                 goto done;
5797
5798         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5799         if (sb == NULL) {
5800                 rc = ENOMEM;
5801                 goto done;
5802         }
5803         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5804             "Seq#", "Tstamp", "Level", "Facility", "Message");
5805
5806         i = first;
5807         do {
5808                 e = &buf[i];
5809                 if (e->timestamp == 0)
5810                         break;  /* end */
5811
5812                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5813                     e->seqno, e->timestamp,
5814                     (e->level < nitems(devlog_level_strings) ?
5815                         devlog_level_strings[e->level] : "UNKNOWN"),
5816                     (e->facility < nitems(devlog_facility_strings) ?
5817                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5818                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5819                     e->params[2], e->params[3], e->params[4],
5820                     e->params[5], e->params[6], e->params[7]);
5821
5822                 if (++i == nentries)
5823                         i = 0;
5824         } while (i != first);
5825
5826         rc = sbuf_finish(sb);
5827         sbuf_delete(sb);
5828 done:
5829         free(buf, M_CXGBE);
5830         return (rc);
5831 }
5832
5833 static int
5834 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5835 {
5836         struct adapter *sc = arg1;
5837         struct sbuf *sb;
5838         int rc;
5839         struct tp_fcoe_stats stats[4];
5840
5841         rc = sysctl_wire_old_buffer(req, 0);
5842         if (rc != 0)
5843                 return (rc);
5844
5845         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5846         if (sb == NULL)
5847                 return (ENOMEM);
5848
5849         t4_get_fcoe_stats(sc, 0, &stats[0]);
5850         t4_get_fcoe_stats(sc, 1, &stats[1]);
5851         t4_get_fcoe_stats(sc, 2, &stats[2]);
5852         t4_get_fcoe_stats(sc, 3, &stats[3]);
5853
5854         sbuf_printf(sb, "                   channel 0        channel 1        "
5855             "channel 2        channel 3\n");
5856         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5857             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5858             stats[3].octetsDDP);
5859         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5860             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5861         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5862             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5863             stats[3].framesDrop);
5864
5865         rc = sbuf_finish(sb);
5866         sbuf_delete(sb);
5867
5868         return (rc);
5869 }
5870
5871 static int
5872 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5873 {
5874         struct adapter *sc = arg1;
5875         struct sbuf *sb;
5876         int rc, i;
5877         unsigned int map, kbps, ipg, mode;
5878         unsigned int pace_tab[NTX_SCHED];
5879
5880         rc = sysctl_wire_old_buffer(req, 0);
5881         if (rc != 0)
5882                 return (rc);
5883
5884         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5885         if (sb == NULL)
5886                 return (ENOMEM);
5887
5888         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5889         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5890         t4_read_pace_tbl(sc, pace_tab);
5891
5892         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5893             "Class IPG (0.1 ns)   Flow IPG (us)");
5894
5895         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5896                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5897                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5898                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5899                 if (kbps)
5900                         sbuf_printf(sb, "%9u     ", kbps);
5901                 else
5902                         sbuf_printf(sb, " disabled     ");
5903
5904                 if (ipg)
5905                         sbuf_printf(sb, "%13u        ", ipg);
5906                 else
5907                         sbuf_printf(sb, "     disabled        ");
5908
5909                 if (pace_tab[i])
5910                         sbuf_printf(sb, "%10u", pace_tab[i]);
5911                 else
5912                         sbuf_printf(sb, "  disabled");
5913         }
5914
5915         rc = sbuf_finish(sb);
5916         sbuf_delete(sb);
5917
5918         return (rc);
5919 }
5920
5921 static int
5922 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5923 {
5924         struct adapter *sc = arg1;
5925         struct sbuf *sb;
5926         int rc, i, j;
5927         uint64_t *p0, *p1;
5928         struct lb_port_stats s[2];
5929         static const char *stat_name[] = {
5930                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5931                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5932                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5933                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5934                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5935                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5936                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5937         };
5938
5939         rc = sysctl_wire_old_buffer(req, 0);
5940         if (rc != 0)
5941                 return (rc);
5942
5943         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5944         if (sb == NULL)
5945                 return (ENOMEM);
5946
5947         memset(s, 0, sizeof(s));
5948
5949         for (i = 0; i < 4; i += 2) {
5950                 t4_get_lb_stats(sc, i, &s[0]);
5951                 t4_get_lb_stats(sc, i + 1, &s[1]);
5952
5953                 p0 = &s[0].octets;
5954                 p1 = &s[1].octets;
5955                 sbuf_printf(sb, "%s                       Loopback %u"
5956                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5957
5958                 for (j = 0; j < nitems(stat_name); j++)
5959                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5960                                    *p0++, *p1++);
5961         }
5962
5963         rc = sbuf_finish(sb);
5964         sbuf_delete(sb);
5965
5966         return (rc);
5967 }
5968
5969 static int
5970 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5971 {
5972         int rc = 0;
5973         struct port_info *pi = arg1;
5974         struct sbuf *sb;
5975         static const char *linkdnreasons[] = {
5976                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5977                 "PHY overheated", "unknown", "rx los", "reserved7"
5978         };
5979
5980         rc = sysctl_wire_old_buffer(req, 0);
5981         if (rc != 0)
5982                 return(rc);
5983         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5984         if (sb == NULL)
5985                 return (ENOMEM);
5986
5987         if (pi->linkdnrc < 0)
5988                 sbuf_printf(sb, "n/a");
5989         else if (pi->linkdnrc < nitems(linkdnreasons))
5990                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5991         else
5992                 sbuf_printf(sb, "%d", pi->linkdnrc);
5993
5994         rc = sbuf_finish(sb);
5995         sbuf_delete(sb);
5996
5997         return (rc);
5998 }
5999
6000 struct mem_desc {
6001         unsigned int base;
6002         unsigned int limit;
6003         unsigned int idx;
6004 };
6005
6006 static int
6007 mem_desc_cmp(const void *a, const void *b)
6008 {
6009         return ((const struct mem_desc *)a)->base -
6010                ((const struct mem_desc *)b)->base;
6011 }
6012
6013 static void
6014 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6015     unsigned int to)
6016 {
6017         unsigned int size;
6018
6019         size = to - from + 1;
6020         if (size == 0)
6021                 return;
6022
6023         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6024         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6025 }
6026
6027 static int
6028 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6029 {
6030         struct adapter *sc = arg1;
6031         struct sbuf *sb;
6032         int rc, i, n;
6033         uint32_t lo, hi, used, alloc;
6034         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6035         static const char *region[] = {
6036                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6037                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6038                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6039                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6040                 "RQUDP region:", "PBL region:", "TXPBL region:",
6041                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6042                 "On-chip queues:"
6043         };
6044         struct mem_desc avail[4];
6045         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6046         struct mem_desc *md = mem;
6047
6048         rc = sysctl_wire_old_buffer(req, 0);
6049         if (rc != 0)
6050                 return (rc);
6051
6052         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6053         if (sb == NULL)
6054                 return (ENOMEM);
6055
6056         for (i = 0; i < nitems(mem); i++) {
6057                 mem[i].limit = 0;
6058                 mem[i].idx = i;
6059         }
6060
6061         /* Find and sort the populated memory ranges */
6062         i = 0;
6063         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6064         if (lo & F_EDRAM0_ENABLE) {
6065                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6066                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6067                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6068                 avail[i].idx = 0;
6069                 i++;
6070         }
6071         if (lo & F_EDRAM1_ENABLE) {
6072                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6073                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6074                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6075                 avail[i].idx = 1;
6076                 i++;
6077         }
6078         if (lo & F_EXT_MEM_ENABLE) {
6079                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6080                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6081                 avail[i].limit = avail[i].base +
6082                     (G_EXT_MEM_SIZE(hi) << 20);
6083                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6084                 i++;
6085         }
6086         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6087                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6088                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6089                 avail[i].limit = avail[i].base +
6090                     (G_EXT_MEM1_SIZE(hi) << 20);
6091                 avail[i].idx = 4;
6092                 i++;
6093         }
6094         if (!i)                                    /* no memory available */
6095                 return 0;
6096         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6097
6098         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6099         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6100         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6101         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6102         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6103         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6104         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6105         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6106         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6107
6108         /* the next few have explicit upper bounds */
6109         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6110         md->limit = md->base - 1 +
6111                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6112                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6113         md++;
6114
6115         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6116         md->limit = md->base - 1 +
6117                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6118                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6119         md++;
6120
6121         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6122                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6123                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6124                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6125         } else {
6126                 md->base = 0;
6127                 md->idx = nitems(region);  /* hide it */
6128         }
6129         md++;
6130
6131 #define ulp_region(reg) \
6132         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6133         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6134
6135         ulp_region(RX_ISCSI);
6136         ulp_region(RX_TDDP);
6137         ulp_region(TX_TPT);
6138         ulp_region(RX_STAG);
6139         ulp_region(RX_RQ);
6140         ulp_region(RX_RQUDP);
6141         ulp_region(RX_PBL);
6142         ulp_region(TX_PBL);
6143 #undef ulp_region
6144
6145         md->base = 0;
6146         md->idx = nitems(region);
6147         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6148                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6149                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6150                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6151         }
6152         md++;
6153
6154         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6155         md->limit = md->base + sc->tids.ntids - 1;
6156         md++;
6157         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6158         md->limit = md->base + sc->tids.ntids - 1;
6159         md++;
6160
6161         md->base = sc->vres.ocq.start;
6162         if (sc->vres.ocq.size)
6163                 md->limit = md->base + sc->vres.ocq.size - 1;
6164         else
6165                 md->idx = nitems(region);  /* hide it */
6166         md++;
6167
6168         /* add any address-space holes, there can be up to 3 */
6169         for (n = 0; n < i - 1; n++)
6170                 if (avail[n].limit < avail[n + 1].base)
6171                         (md++)->base = avail[n].limit;
6172         if (avail[n].limit)
6173                 (md++)->base = avail[n].limit;
6174
6175         n = md - mem;
6176         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6177
6178         for (lo = 0; lo < i; lo++)
6179                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6180                                 avail[lo].limit - 1);
6181
6182         sbuf_printf(sb, "\n");
6183         for (i = 0; i < n; i++) {
6184                 if (mem[i].idx >= nitems(region))
6185                         continue;                        /* skip holes */
6186                 if (!mem[i].limit)
6187                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6188                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6189                                 mem[i].limit);
6190         }
6191
6192         sbuf_printf(sb, "\n");
6193         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6194         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6195         mem_region_show(sb, "uP RAM:", lo, hi);
6196
6197         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6198         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6199         mem_region_show(sb, "uP Extmem2:", lo, hi);
6200
6201         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6202         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6203                    G_PMRXMAXPAGE(lo),
6204                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6205                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6206
6207         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6208         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6209         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6210                    G_PMTXMAXPAGE(lo),
6211                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6212                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6213         sbuf_printf(sb, "%u p-structs\n",
6214                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6215
6216         for (i = 0; i < 4; i++) {
6217                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6218                 if (is_t4(sc)) {
6219                         used = G_USED(lo);
6220                         alloc = G_ALLOC(lo);
6221                 } else {
6222                         used = G_T5_USED(lo);
6223                         alloc = G_T5_ALLOC(lo);
6224                 }
6225                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6226                            i, used, alloc);
6227         }
6228         for (i = 0; i < 4; i++) {
6229                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6230                 if (is_t4(sc)) {
6231                         used = G_USED(lo);
6232                         alloc = G_ALLOC(lo);
6233                 } else {
6234                         used = G_T5_USED(lo);
6235                         alloc = G_T5_ALLOC(lo);
6236                 }
6237                 sbuf_printf(sb,
6238                            "\nLoopback %d using %u pages out of %u allocated",
6239                            i, used, alloc);
6240         }
6241
6242         rc = sbuf_finish(sb);
6243         sbuf_delete(sb);
6244
6245         return (rc);
6246 }
6247
6248 static inline void
6249 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6250 {
6251         *mask = x | y;
6252         y = htobe64(y);
6253         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6254 }
6255
6256 static int
6257 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6258 {
6259         struct adapter *sc = arg1;
6260         struct sbuf *sb;
6261         int rc, i, n;
6262
6263         rc = sysctl_wire_old_buffer(req, 0);
6264         if (rc != 0)
6265                 return (rc);
6266
6267         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6268         if (sb == NULL)
6269                 return (ENOMEM);
6270
6271         sbuf_printf(sb,
6272             "Idx  Ethernet address     Mask     Vld Ports PF"
6273             "  VF              Replication             P0 P1 P2 P3  ML");
6274         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6275             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6276         for (i = 0; i < n; i++) {
6277                 uint64_t tcamx, tcamy, mask;
6278                 uint32_t cls_lo, cls_hi;
6279                 uint8_t addr[ETHER_ADDR_LEN];
6280
6281                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6282                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6283                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6284                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6285
6286                 if (tcamx & tcamy)
6287                         continue;
6288
6289                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6290                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6291                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6292                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6293                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6294                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6295                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6296
6297                 if (cls_lo & F_REPLICATE) {
6298                         struct fw_ldst_cmd ldst_cmd;
6299
6300                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6301                         ldst_cmd.op_to_addrspace =
6302                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6303                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6304                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6305                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6306                         ldst_cmd.u.mps.rplc.fid_idx =
6307                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6308                                 V_FW_LDST_CMD_IDX(i));
6309
6310                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6311                             "t4mps");
6312                         if (rc)
6313                                 break;
6314                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6315                             sizeof(ldst_cmd), &ldst_cmd);
6316                         end_synchronized_op(sc, 0);
6317
6318                         if (rc != 0) {
6319                                 sbuf_printf(sb,
6320                                     " ------------ error %3u ------------", rc);
6321                                 rc = 0;
6322                         } else {
6323                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6324                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6325                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6326                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6327                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6328                         }
6329                 } else
6330                         sbuf_printf(sb, "%36s", "");
6331
6332                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6333                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6334                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6335         }
6336
6337         if (rc)
6338                 (void) sbuf_finish(sb);
6339         else
6340                 rc = sbuf_finish(sb);
6341         sbuf_delete(sb);
6342
6343         return (rc);
6344 }
6345
6346 static int
6347 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6348 {
6349         struct adapter *sc = arg1;
6350         struct sbuf *sb;
6351         int rc;
6352         uint16_t mtus[NMTUS];
6353
6354         rc = sysctl_wire_old_buffer(req, 0);
6355         if (rc != 0)
6356                 return (rc);
6357
6358         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6359         if (sb == NULL)
6360                 return (ENOMEM);
6361
6362         t4_read_mtu_tbl(sc, mtus, NULL);
6363
6364         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6365             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6366             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6367             mtus[14], mtus[15]);
6368
6369         rc = sbuf_finish(sb);
6370         sbuf_delete(sb);
6371
6372         return (rc);
6373 }
6374
6375 static int
6376 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6377 {
6378         struct adapter *sc = arg1;
6379         struct sbuf *sb;
6380         int rc, i;
6381         uint32_t cnt[PM_NSTATS];
6382         uint64_t cyc[PM_NSTATS];
6383         static const char *rx_stats[] = {
6384                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6385         };
6386         static const char *tx_stats[] = {
6387                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6388         };
6389
6390         rc = sysctl_wire_old_buffer(req, 0);
6391         if (rc != 0)
6392                 return (rc);
6393
6394         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6395         if (sb == NULL)
6396                 return (ENOMEM);
6397
6398         t4_pmtx_get_stats(sc, cnt, cyc);
6399         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6400         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6401                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6402                     cyc[i]);
6403
6404         t4_pmrx_get_stats(sc, cnt, cyc);
6405         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6406         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6407                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6408                     cyc[i]);
6409
6410         rc = sbuf_finish(sb);
6411         sbuf_delete(sb);
6412
6413         return (rc);
6414 }
6415
6416 static int
6417 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6418 {
6419         struct adapter *sc = arg1;
6420         struct sbuf *sb;
6421         int rc;
6422         struct tp_rdma_stats stats;
6423
6424         rc = sysctl_wire_old_buffer(req, 0);
6425         if (rc != 0)
6426                 return (rc);
6427
6428         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6429         if (sb == NULL)
6430                 return (ENOMEM);
6431
6432         t4_tp_get_rdma_stats(sc, &stats);
6433         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6434         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6435
6436         rc = sbuf_finish(sb);
6437         sbuf_delete(sb);
6438
6439         return (rc);
6440 }
6441
6442 static int
6443 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6444 {
6445         struct adapter *sc = arg1;
6446         struct sbuf *sb;
6447         int rc;
6448         struct tp_tcp_stats v4, v6;
6449
6450         rc = sysctl_wire_old_buffer(req, 0);
6451         if (rc != 0)
6452                 return (rc);
6453
6454         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6455         if (sb == NULL)
6456                 return (ENOMEM);
6457
6458         t4_tp_get_tcp_stats(sc, &v4, &v6);
6459         sbuf_printf(sb,
6460             "                                IP                 IPv6\n");
6461         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6462             v4.tcpOutRsts, v6.tcpOutRsts);
6463         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6464             v4.tcpInSegs, v6.tcpInSegs);
6465         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6466             v4.tcpOutSegs, v6.tcpOutSegs);
6467         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6468             v4.tcpRetransSegs, v6.tcpRetransSegs);
6469
6470         rc = sbuf_finish(sb);
6471         sbuf_delete(sb);
6472
6473         return (rc);
6474 }
6475
6476 static int
6477 sysctl_tids(SYSCTL_HANDLER_ARGS)
6478 {
6479         struct adapter *sc = arg1;
6480         struct sbuf *sb;
6481         int rc;
6482         struct tid_info *t = &sc->tids;
6483
6484         rc = sysctl_wire_old_buffer(req, 0);
6485         if (rc != 0)
6486                 return (rc);
6487
6488         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6489         if (sb == NULL)
6490                 return (ENOMEM);
6491
6492         if (t->natids) {
6493                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6494                     t->atids_in_use);
6495         }
6496
6497         if (t->ntids) {
6498                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6499                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6500
6501                         if (b) {
6502                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6503                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6504                                     t->ntids - 1);
6505                         } else {
6506                                 sbuf_printf(sb, "TID range: %u-%u",
6507                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6508                                     t->ntids - 1);
6509                         }
6510                 } else
6511                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6512                 sbuf_printf(sb, ", in use: %u\n",
6513                     atomic_load_acq_int(&t->tids_in_use));
6514         }
6515
6516         if (t->nstids) {
6517                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6518                     t->stid_base + t->nstids - 1, t->stids_in_use);
6519         }
6520
6521         if (t->nftids) {
6522                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6523                     t->ftid_base + t->nftids - 1);
6524         }
6525
6526         if (t->netids) {
6527                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6528                     t->etid_base + t->netids - 1);
6529         }
6530
6531         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6532             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6533             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6534
6535         rc = sbuf_finish(sb);
6536         sbuf_delete(sb);
6537
6538         return (rc);
6539 }
6540
6541 static int
6542 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6543 {
6544         struct adapter *sc = arg1;
6545         struct sbuf *sb;
6546         int rc;
6547         struct tp_err_stats stats;
6548
6549         rc = sysctl_wire_old_buffer(req, 0);
6550         if (rc != 0)
6551                 return (rc);
6552
6553         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6554         if (sb == NULL)
6555                 return (ENOMEM);
6556
6557         t4_tp_get_err_stats(sc, &stats);
6558
6559         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6560                       "channel 3\n");
6561         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6562             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6563             stats.macInErrs[3]);
6564         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6565             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6566             stats.hdrInErrs[3]);
6567         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6568             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6569             stats.tcpInErrs[3]);
6570         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6571             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6572             stats.tcp6InErrs[3]);
6573         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6574             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6575             stats.tnlCongDrops[3]);
6576         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6577             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6578             stats.tnlTxDrops[3]);
6579         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6580             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6581             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6582         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6583             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6584             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6585         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6586             stats.ofldNoNeigh, stats.ofldCongDefer);
6587
6588         rc = sbuf_finish(sb);
6589         sbuf_delete(sb);
6590
6591         return (rc);
6592 }
6593
6594 struct field_desc {
6595         const char *name;
6596         u_int start;
6597         u_int width;
6598 };
6599
6600 static void
6601 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6602 {
6603         char buf[32];
6604         int line_size = 0;
6605
6606         while (f->name) {
6607                 uint64_t mask = (1ULL << f->width) - 1;
6608                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6609                     ((uintmax_t)v >> f->start) & mask);
6610
6611                 if (line_size + len >= 79) {
6612                         line_size = 8;
6613                         sbuf_printf(sb, "\n        ");
6614                 }
6615                 sbuf_printf(sb, "%s ", buf);
6616                 line_size += len + 1;
6617                 f++;
6618         }
6619         sbuf_printf(sb, "\n");
6620 }
6621
6622 static struct field_desc tp_la0[] = {
6623         { "RcfOpCodeOut", 60, 4 },
6624         { "State", 56, 4 },
6625         { "WcfState", 52, 4 },
6626         { "RcfOpcSrcOut", 50, 2 },
6627         { "CRxError", 49, 1 },
6628         { "ERxError", 48, 1 },
6629         { "SanityFailed", 47, 1 },
6630         { "SpuriousMsg", 46, 1 },
6631         { "FlushInputMsg", 45, 1 },
6632         { "FlushInputCpl", 44, 1 },
6633         { "RssUpBit", 43, 1 },
6634         { "RssFilterHit", 42, 1 },
6635         { "Tid", 32, 10 },
6636         { "InitTcb", 31, 1 },
6637         { "LineNumber", 24, 7 },
6638         { "Emsg", 23, 1 },
6639         { "EdataOut", 22, 1 },
6640         { "Cmsg", 21, 1 },
6641         { "CdataOut", 20, 1 },
6642         { "EreadPdu", 19, 1 },
6643         { "CreadPdu", 18, 1 },
6644         { "TunnelPkt", 17, 1 },
6645         { "RcfPeerFin", 16, 1 },
6646         { "RcfReasonOut", 12, 4 },
6647         { "TxCchannel", 10, 2 },
6648         { "RcfTxChannel", 8, 2 },
6649         { "RxEchannel", 6, 2 },
6650         { "RcfRxChannel", 5, 1 },
6651         { "RcfDataOutSrdy", 4, 1 },
6652         { "RxDvld", 3, 1 },
6653         { "RxOoDvld", 2, 1 },
6654         { "RxCongestion", 1, 1 },
6655         { "TxCongestion", 0, 1 },
6656         { NULL }
6657 };
6658
6659 static struct field_desc tp_la1[] = {
6660         { "CplCmdIn", 56, 8 },
6661         { "CplCmdOut", 48, 8 },
6662         { "ESynOut", 47, 1 },
6663         { "EAckOut", 46, 1 },
6664         { "EFinOut", 45, 1 },
6665         { "ERstOut", 44, 1 },
6666         { "SynIn", 43, 1 },
6667         { "AckIn", 42, 1 },
6668         { "FinIn", 41, 1 },
6669         { "RstIn", 40, 1 },
6670         { "DataIn", 39, 1 },
6671         { "DataInVld", 38, 1 },
6672         { "PadIn", 37, 1 },
6673         { "RxBufEmpty", 36, 1 },
6674         { "RxDdp", 35, 1 },
6675         { "RxFbCongestion", 34, 1 },
6676         { "TxFbCongestion", 33, 1 },
6677         { "TxPktSumSrdy", 32, 1 },
6678         { "RcfUlpType", 28, 4 },
6679         { "Eread", 27, 1 },
6680         { "Ebypass", 26, 1 },
6681         { "Esave", 25, 1 },
6682         { "Static0", 24, 1 },
6683         { "Cread", 23, 1 },
6684         { "Cbypass", 22, 1 },
6685         { "Csave", 21, 1 },
6686         { "CPktOut", 20, 1 },
6687         { "RxPagePoolFull", 18, 2 },
6688         { "RxLpbkPkt", 17, 1 },
6689         { "TxLpbkPkt", 16, 1 },
6690         { "RxVfValid", 15, 1 },
6691         { "SynLearned", 14, 1 },
6692         { "SetDelEntry", 13, 1 },
6693         { "SetInvEntry", 12, 1 },
6694         { "CpcmdDvld", 11, 1 },
6695         { "CpcmdSave", 10, 1 },
6696         { "RxPstructsFull", 8, 2 },
6697         { "EpcmdDvld", 7, 1 },
6698         { "EpcmdFlush", 6, 1 },
6699         { "EpcmdTrimPrefix", 5, 1 },
6700         { "EpcmdTrimPostfix", 4, 1 },
6701         { "ERssIp4Pkt", 3, 1 },
6702         { "ERssIp6Pkt", 2, 1 },
6703         { "ERssTcpUdpPkt", 1, 1 },
6704         { "ERssFceFipPkt", 0, 1 },
6705         { NULL }
6706 };
6707
6708 static struct field_desc tp_la2[] = {
6709         { "CplCmdIn", 56, 8 },
6710         { "MpsVfVld", 55, 1 },
6711         { "MpsPf", 52, 3 },
6712         { "MpsVf", 44, 8 },
6713         { "SynIn", 43, 1 },
6714         { "AckIn", 42, 1 },
6715         { "FinIn", 41, 1 },
6716         { "RstIn", 40, 1 },
6717         { "DataIn", 39, 1 },
6718         { "DataInVld", 38, 1 },
6719         { "PadIn", 37, 1 },
6720         { "RxBufEmpty", 36, 1 },
6721         { "RxDdp", 35, 1 },
6722         { "RxFbCongestion", 34, 1 },
6723         { "TxFbCongestion", 33, 1 },
6724         { "TxPktSumSrdy", 32, 1 },
6725         { "RcfUlpType", 28, 4 },
6726         { "Eread", 27, 1 },
6727         { "Ebypass", 26, 1 },
6728         { "Esave", 25, 1 },
6729         { "Static0", 24, 1 },
6730         { "Cread", 23, 1 },
6731         { "Cbypass", 22, 1 },
6732         { "Csave", 21, 1 },
6733         { "CPktOut", 20, 1 },
6734         { "RxPagePoolFull", 18, 2 },
6735         { "RxLpbkPkt", 17, 1 },
6736         { "TxLpbkPkt", 16, 1 },
6737         { "RxVfValid", 15, 1 },
6738         { "SynLearned", 14, 1 },
6739         { "SetDelEntry", 13, 1 },
6740         { "SetInvEntry", 12, 1 },
6741         { "CpcmdDvld", 11, 1 },
6742         { "CpcmdSave", 10, 1 },
6743         { "RxPstructsFull", 8, 2 },
6744         { "EpcmdDvld", 7, 1 },
6745         { "EpcmdFlush", 6, 1 },
6746         { "EpcmdTrimPrefix", 5, 1 },
6747         { "EpcmdTrimPostfix", 4, 1 },
6748         { "ERssIp4Pkt", 3, 1 },
6749         { "ERssIp6Pkt", 2, 1 },
6750         { "ERssTcpUdpPkt", 1, 1 },
6751         { "ERssFceFipPkt", 0, 1 },
6752         { NULL }
6753 };
6754
6755 static void
6756 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6757 {
6758
6759         field_desc_show(sb, *p, tp_la0);
6760 }
6761
6762 static void
6763 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6764 {
6765
6766         if (idx)
6767                 sbuf_printf(sb, "\n");
6768         field_desc_show(sb, p[0], tp_la0);
6769         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6770                 field_desc_show(sb, p[1], tp_la0);
6771 }
6772
6773 static void
6774 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6775 {
6776
6777         if (idx)
6778                 sbuf_printf(sb, "\n");
6779         field_desc_show(sb, p[0], tp_la0);
6780         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6781                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6782 }
6783
6784 static int
6785 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6786 {
6787         struct adapter *sc = arg1;
6788         struct sbuf *sb;
6789         uint64_t *buf, *p;
6790         int rc;
6791         u_int i, inc;
6792         void (*show_func)(struct sbuf *, uint64_t *, int);
6793
6794         rc = sysctl_wire_old_buffer(req, 0);
6795         if (rc != 0)
6796                 return (rc);
6797
6798         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6799         if (sb == NULL)
6800                 return (ENOMEM);
6801
6802         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6803
6804         t4_tp_read_la(sc, buf, NULL);
6805         p = buf;
6806
6807         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6808         case 2:
6809                 inc = 2;
6810                 show_func = tp_la_show2;
6811                 break;
6812         case 3:
6813                 inc = 2;
6814                 show_func = tp_la_show3;
6815                 break;
6816         default:
6817                 inc = 1;
6818                 show_func = tp_la_show;
6819         }
6820
6821         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6822                 (*show_func)(sb, p, i);
6823
6824         rc = sbuf_finish(sb);
6825         sbuf_delete(sb);
6826         free(buf, M_CXGBE);
6827         return (rc);
6828 }
6829
6830 static int
6831 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6832 {
6833         struct adapter *sc = arg1;
6834         struct sbuf *sb;
6835         int rc;
6836         u64 nrate[NCHAN], orate[NCHAN];
6837
6838         rc = sysctl_wire_old_buffer(req, 0);
6839         if (rc != 0)
6840                 return (rc);
6841
6842         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6843         if (sb == NULL)
6844                 return (ENOMEM);
6845
6846         t4_get_chan_txrate(sc, nrate, orate);
6847         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6848                  "channel 3\n");
6849         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6850             nrate[0], nrate[1], nrate[2], nrate[3]);
6851         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6852             orate[0], orate[1], orate[2], orate[3]);
6853
6854         rc = sbuf_finish(sb);
6855         sbuf_delete(sb);
6856
6857         return (rc);
6858 }
6859
6860 static int
6861 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6862 {
6863         struct adapter *sc = arg1;
6864         struct sbuf *sb;
6865         uint32_t *buf, *p;
6866         int rc, i;
6867
6868         rc = sysctl_wire_old_buffer(req, 0);
6869         if (rc != 0)
6870                 return (rc);
6871
6872         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6873         if (sb == NULL)
6874                 return (ENOMEM);
6875
6876         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6877             M_ZERO | M_WAITOK);
6878
6879         t4_ulprx_read_la(sc, buf);
6880         p = buf;
6881
6882         sbuf_printf(sb, "      Pcmd        Type   Message"
6883             "                Data");
6884         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6885                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6886                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6887         }
6888
6889         rc = sbuf_finish(sb);
6890         sbuf_delete(sb);
6891         free(buf, M_CXGBE);
6892         return (rc);
6893 }
6894
6895 static int
6896 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6897 {
6898         struct adapter *sc = arg1;
6899         struct sbuf *sb;
6900         int rc, v;
6901
6902         rc = sysctl_wire_old_buffer(req, 0);
6903         if (rc != 0)
6904                 return (rc);
6905
6906         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6907         if (sb == NULL)
6908                 return (ENOMEM);
6909
6910         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6911         if (G_STATSOURCE_T5(v) == 7) {
6912                 if (G_STATMODE(v) == 0) {
6913                         sbuf_printf(sb, "total %d, incomplete %d",
6914                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6915                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6916                 } else if (G_STATMODE(v) == 1) {
6917                         sbuf_printf(sb, "total %d, data overflow %d",
6918                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6919                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6920                 }
6921         }
6922         rc = sbuf_finish(sb);
6923         sbuf_delete(sb);
6924
6925         return (rc);
6926 }
6927 #endif
6928
6929 static uint32_t
6930 fconf_to_mode(uint32_t fconf)
6931 {
6932         uint32_t mode;
6933
6934         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6935             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6936
6937         if (fconf & F_FRAGMENTATION)
6938                 mode |= T4_FILTER_IP_FRAGMENT;
6939
6940         if (fconf & F_MPSHITTYPE)
6941                 mode |= T4_FILTER_MPS_HIT_TYPE;
6942
6943         if (fconf & F_MACMATCH)
6944                 mode |= T4_FILTER_MAC_IDX;
6945
6946         if (fconf & F_ETHERTYPE)
6947                 mode |= T4_FILTER_ETH_TYPE;
6948
6949         if (fconf & F_PROTOCOL)
6950                 mode |= T4_FILTER_IP_PROTO;
6951
6952         if (fconf & F_TOS)
6953                 mode |= T4_FILTER_IP_TOS;
6954
6955         if (fconf & F_VLAN)
6956                 mode |= T4_FILTER_VLAN;
6957
6958         if (fconf & F_VNIC_ID)
6959                 mode |= T4_FILTER_VNIC;
6960
6961         if (fconf & F_PORT)
6962                 mode |= T4_FILTER_PORT;
6963
6964         if (fconf & F_FCOE)
6965                 mode |= T4_FILTER_FCoE;
6966
6967         return (mode);
6968 }
6969
6970 static uint32_t
6971 mode_to_fconf(uint32_t mode)
6972 {
6973         uint32_t fconf = 0;
6974
6975         if (mode & T4_FILTER_IP_FRAGMENT)
6976                 fconf |= F_FRAGMENTATION;
6977
6978         if (mode & T4_FILTER_MPS_HIT_TYPE)
6979                 fconf |= F_MPSHITTYPE;
6980
6981         if (mode & T4_FILTER_MAC_IDX)
6982                 fconf |= F_MACMATCH;
6983
6984         if (mode & T4_FILTER_ETH_TYPE)
6985                 fconf |= F_ETHERTYPE;
6986
6987         if (mode & T4_FILTER_IP_PROTO)
6988                 fconf |= F_PROTOCOL;
6989
6990         if (mode & T4_FILTER_IP_TOS)
6991                 fconf |= F_TOS;
6992
6993         if (mode & T4_FILTER_VLAN)
6994                 fconf |= F_VLAN;
6995
6996         if (mode & T4_FILTER_VNIC)
6997                 fconf |= F_VNIC_ID;
6998
6999         if (mode & T4_FILTER_PORT)
7000                 fconf |= F_PORT;
7001
7002         if (mode & T4_FILTER_FCoE)
7003                 fconf |= F_FCOE;
7004
7005         return (fconf);
7006 }
7007
7008 static uint32_t
7009 fspec_to_fconf(struct t4_filter_specification *fs)
7010 {
7011         uint32_t fconf = 0;
7012
7013         if (fs->val.frag || fs->mask.frag)
7014                 fconf |= F_FRAGMENTATION;
7015
7016         if (fs->val.matchtype || fs->mask.matchtype)
7017                 fconf |= F_MPSHITTYPE;
7018
7019         if (fs->val.macidx || fs->mask.macidx)
7020                 fconf |= F_MACMATCH;
7021
7022         if (fs->val.ethtype || fs->mask.ethtype)
7023                 fconf |= F_ETHERTYPE;
7024
7025         if (fs->val.proto || fs->mask.proto)
7026                 fconf |= F_PROTOCOL;
7027
7028         if (fs->val.tos || fs->mask.tos)
7029                 fconf |= F_TOS;
7030
7031         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7032                 fconf |= F_VLAN;
7033
7034         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7035                 fconf |= F_VNIC_ID;
7036
7037         if (fs->val.iport || fs->mask.iport)
7038                 fconf |= F_PORT;
7039
7040         if (fs->val.fcoe || fs->mask.fcoe)
7041                 fconf |= F_FCOE;
7042
7043         return (fconf);
7044 }
7045
7046 static int
7047 get_filter_mode(struct adapter *sc, uint32_t *mode)
7048 {
7049         int rc;
7050         uint32_t fconf;
7051
7052         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7053             "t4getfm");
7054         if (rc)
7055                 return (rc);
7056
7057         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7058             A_TP_VLAN_PRI_MAP);
7059
7060         if (sc->params.tp.vlan_pri_map != fconf) {
7061                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7062                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7063                     fconf);
7064         }
7065
7066         *mode = fconf_to_mode(fconf);
7067
7068         end_synchronized_op(sc, LOCK_HELD);
7069         return (0);
7070 }
7071
7072 static int
7073 set_filter_mode(struct adapter *sc, uint32_t mode)
7074 {
7075         uint32_t fconf;
7076         int rc;
7077
7078         fconf = mode_to_fconf(mode);
7079
7080         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7081             "t4setfm");
7082         if (rc)
7083                 return (rc);
7084
7085         if (sc->tids.ftids_in_use > 0) {
7086                 rc = EBUSY;
7087                 goto done;
7088         }
7089
7090 #ifdef TCP_OFFLOAD
7091         if (uld_active(sc, ULD_TOM)) {
7092                 rc = EBUSY;
7093                 goto done;
7094         }
7095 #endif
7096
7097         rc = -t4_set_filter_mode(sc, fconf);
7098 done:
7099         end_synchronized_op(sc, LOCK_HELD);
7100         return (rc);
7101 }
7102
7103 static inline uint64_t
7104 get_filter_hits(struct adapter *sc, uint32_t fid)
7105 {
7106         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7107         uint64_t hits;
7108
7109         memwin_info(sc, 0, &mw_base, NULL);
7110         off = position_memwin(sc, 0,
7111             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7112         if (is_t4(sc)) {
7113                 hits = t4_read_reg64(sc, mw_base + off + 16);
7114                 hits = be64toh(hits);
7115         } else {
7116                 hits = t4_read_reg(sc, mw_base + off + 24);
7117                 hits = be32toh(hits);
7118         }
7119
7120         return (hits);
7121 }
7122
7123 static int
7124 get_filter(struct adapter *sc, struct t4_filter *t)
7125 {
7126         int i, rc, nfilters = sc->tids.nftids;
7127         struct filter_entry *f;
7128
7129         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7130             "t4getf");
7131         if (rc)
7132                 return (rc);
7133
7134         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7135             t->idx >= nfilters) {
7136                 t->idx = 0xffffffff;
7137                 goto done;
7138         }
7139
7140         f = &sc->tids.ftid_tab[t->idx];
7141         for (i = t->idx; i < nfilters; i++, f++) {
7142                 if (f->valid) {
7143                         t->idx = i;
7144                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7145                         t->smtidx = f->smtidx;
7146                         if (f->fs.hitcnts)
7147                                 t->hits = get_filter_hits(sc, t->idx);
7148                         else
7149                                 t->hits = UINT64_MAX;
7150                         t->fs = f->fs;
7151
7152                         goto done;
7153                 }
7154         }
7155
7156         t->idx = 0xffffffff;
7157 done:
7158         end_synchronized_op(sc, LOCK_HELD);
7159         return (0);
7160 }
7161
7162 static int
7163 set_filter(struct adapter *sc, struct t4_filter *t)
7164 {
7165         unsigned int nfilters, nports;
7166         struct filter_entry *f;
7167         int i, rc;
7168
7169         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7170         if (rc)
7171                 return (rc);
7172
7173         nfilters = sc->tids.nftids;
7174         nports = sc->params.nports;
7175
7176         if (nfilters == 0) {
7177                 rc = ENOTSUP;
7178                 goto done;
7179         }
7180
7181         if (!(sc->flags & FULL_INIT_DONE)) {
7182                 rc = EAGAIN;
7183                 goto done;
7184         }
7185
7186         if (t->idx >= nfilters) {
7187                 rc = EINVAL;
7188                 goto done;
7189         }
7190
7191         /* Validate against the global filter mode */
7192         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7193             sc->params.tp.vlan_pri_map) {
7194                 rc = E2BIG;
7195                 goto done;
7196         }
7197
7198         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7199                 rc = EINVAL;
7200                 goto done;
7201         }
7202
7203         if (t->fs.val.iport >= nports) {
7204                 rc = EINVAL;
7205                 goto done;
7206         }
7207
7208         /* Can't specify an iq if not steering to it */
7209         if (!t->fs.dirsteer && t->fs.iq) {
7210                 rc = EINVAL;
7211                 goto done;
7212         }
7213
7214         /* IPv6 filter idx must be 4 aligned */
7215         if (t->fs.type == 1 &&
7216             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7217                 rc = EINVAL;
7218                 goto done;
7219         }
7220
7221         if (sc->tids.ftid_tab == NULL) {
7222                 KASSERT(sc->tids.ftids_in_use == 0,
7223                     ("%s: no memory allocated but filters_in_use > 0",
7224                     __func__));
7225
7226                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7227                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7228                 if (sc->tids.ftid_tab == NULL) {
7229                         rc = ENOMEM;
7230                         goto done;
7231                 }
7232                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7233         }
7234
7235         for (i = 0; i < 4; i++) {
7236                 f = &sc->tids.ftid_tab[t->idx + i];
7237
7238                 if (f->pending || f->valid) {
7239                         rc = EBUSY;
7240                         goto done;
7241                 }
7242                 if (f->locked) {
7243                         rc = EPERM;
7244                         goto done;
7245                 }
7246
7247                 if (t->fs.type == 0)
7248                         break;
7249         }
7250
7251         f = &sc->tids.ftid_tab[t->idx];
7252         f->fs = t->fs;
7253
7254         rc = set_filter_wr(sc, t->idx);
7255 done:
7256         end_synchronized_op(sc, 0);
7257
7258         if (rc == 0) {
7259                 mtx_lock(&sc->tids.ftid_lock);
7260                 for (;;) {
7261                         if (f->pending == 0) {
7262                                 rc = f->valid ? 0 : EIO;
7263                                 break;
7264                         }
7265
7266                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7267                             PCATCH, "t4setfw", 0)) {
7268                                 rc = EINPROGRESS;
7269                                 break;
7270                         }
7271                 }
7272                 mtx_unlock(&sc->tids.ftid_lock);
7273         }
7274         return (rc);
7275 }
7276
7277 static int
7278 del_filter(struct adapter *sc, struct t4_filter *t)
7279 {
7280         unsigned int nfilters;
7281         struct filter_entry *f;
7282         int rc;
7283
7284         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7285         if (rc)
7286                 return (rc);
7287
7288         nfilters = sc->tids.nftids;
7289
7290         if (nfilters == 0) {
7291                 rc = ENOTSUP;
7292                 goto done;
7293         }
7294
7295         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7296             t->idx >= nfilters) {
7297                 rc = EINVAL;
7298                 goto done;
7299         }
7300
7301         if (!(sc->flags & FULL_INIT_DONE)) {
7302                 rc = EAGAIN;
7303                 goto done;
7304         }
7305
7306         f = &sc->tids.ftid_tab[t->idx];
7307
7308         if (f->pending) {
7309                 rc = EBUSY;
7310                 goto done;
7311         }
7312         if (f->locked) {
7313                 rc = EPERM;
7314                 goto done;
7315         }
7316
7317         if (f->valid) {
7318                 t->fs = f->fs;  /* extra info for the caller */
7319                 rc = del_filter_wr(sc, t->idx);
7320         }
7321
7322 done:
7323         end_synchronized_op(sc, 0);
7324
7325         if (rc == 0) {
7326                 mtx_lock(&sc->tids.ftid_lock);
7327                 for (;;) {
7328                         if (f->pending == 0) {
7329                                 rc = f->valid ? EIO : 0;
7330                                 break;
7331                         }
7332
7333                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7334                             PCATCH, "t4delfw", 0)) {
7335                                 rc = EINPROGRESS;
7336                                 break;
7337                         }
7338                 }
7339                 mtx_unlock(&sc->tids.ftid_lock);
7340         }
7341
7342         return (rc);
7343 }
7344
7345 static void
7346 clear_filter(struct filter_entry *f)
7347 {
7348         if (f->l2t)
7349                 t4_l2t_release(f->l2t);
7350
7351         bzero(f, sizeof (*f));
7352 }
7353
7354 static int
7355 set_filter_wr(struct adapter *sc, int fidx)
7356 {
7357         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7358         struct fw_filter_wr *fwr;
7359         unsigned int ftid;
7360         struct wrq_cookie cookie;
7361
7362         ASSERT_SYNCHRONIZED_OP(sc);
7363
7364         if (f->fs.newdmac || f->fs.newvlan) {
7365                 /* This filter needs an L2T entry; allocate one. */
7366                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7367                 if (f->l2t == NULL)
7368                         return (EAGAIN);
7369                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7370                     f->fs.dmac)) {
7371                         t4_l2t_release(f->l2t);
7372                         f->l2t = NULL;
7373                         return (ENOMEM);
7374                 }
7375         }
7376
7377         ftid = sc->tids.ftid_base + fidx;
7378
7379         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7380         if (fwr == NULL)
7381                 return (ENOMEM);
7382         bzero(fwr, sizeof(*fwr));
7383
7384         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7385         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7386         fwr->tid_to_iq =
7387             htobe32(V_FW_FILTER_WR_TID(ftid) |
7388                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7389                 V_FW_FILTER_WR_NOREPLY(0) |
7390                 V_FW_FILTER_WR_IQ(f->fs.iq));
7391         fwr->del_filter_to_l2tix =
7392             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7393                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7394                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7395                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7396                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7397                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7398                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7399                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7400                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7401                     f->fs.newvlan == VLAN_REWRITE) |
7402                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7403                     f->fs.newvlan == VLAN_REWRITE) |
7404                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7405                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7406                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7407                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7408         fwr->ethtype = htobe16(f->fs.val.ethtype);
7409         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7410         fwr->frag_to_ovlan_vldm =
7411             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7412                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7413                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7414                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7415                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7416                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7417         fwr->smac_sel = 0;
7418         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7419             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7420         fwr->maci_to_matchtypem =
7421             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7422                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7423                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7424                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7425                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7426                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7427                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7428                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7429         fwr->ptcl = f->fs.val.proto;
7430         fwr->ptclm = f->fs.mask.proto;
7431         fwr->ttyp = f->fs.val.tos;
7432         fwr->ttypm = f->fs.mask.tos;
7433         fwr->ivlan = htobe16(f->fs.val.vlan);
7434         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7435         fwr->ovlan = htobe16(f->fs.val.vnic);
7436         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7437         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7438         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7439         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7440         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7441         fwr->lp = htobe16(f->fs.val.dport);
7442         fwr->lpm = htobe16(f->fs.mask.dport);
7443         fwr->fp = htobe16(f->fs.val.sport);
7444         fwr->fpm = htobe16(f->fs.mask.sport);
7445         if (f->fs.newsmac)
7446                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7447
7448         f->pending = 1;
7449         sc->tids.ftids_in_use++;
7450
7451         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7452         return (0);
7453 }
7454
7455 static int
7456 del_filter_wr(struct adapter *sc, int fidx)
7457 {
7458         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7459         struct fw_filter_wr *fwr;
7460         unsigned int ftid;
7461         struct wrq_cookie cookie;
7462
7463         ftid = sc->tids.ftid_base + fidx;
7464
7465         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7466         if (fwr == NULL)
7467                 return (ENOMEM);
7468         bzero(fwr, sizeof (*fwr));
7469
7470         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7471
7472         f->pending = 1;
7473         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7474         return (0);
7475 }
7476
7477 int
7478 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7479 {
7480         struct adapter *sc = iq->adapter;
7481         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7482         unsigned int idx = GET_TID(rpl);
7483         unsigned int rc;
7484         struct filter_entry *f;
7485
7486         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7487             rss->opcode));
7488
7489         if (is_ftid(sc, idx)) {
7490
7491                 idx -= sc->tids.ftid_base;
7492                 f = &sc->tids.ftid_tab[idx];
7493                 rc = G_COOKIE(rpl->cookie);
7494
7495                 mtx_lock(&sc->tids.ftid_lock);
7496                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7497                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7498                             __func__, idx));
7499                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7500                         f->pending = 0;  /* asynchronous setup completed */
7501                         f->valid = 1;
7502                 } else {
7503                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7504                                 /* Add or delete failed, display an error */
7505                                 log(LOG_ERR,
7506                                     "filter %u setup failed with error %u\n",
7507                                     idx, rc);
7508                         }
7509
7510                         clear_filter(f);
7511                         sc->tids.ftids_in_use--;
7512                 }
7513                 wakeup(&sc->tids.ftid_tab);
7514                 mtx_unlock(&sc->tids.ftid_lock);
7515         }
7516
7517         return (0);
7518 }
7519
7520 static int
7521 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7522 {
7523         int rc;
7524
7525         if (cntxt->cid > M_CTXTQID)
7526                 return (EINVAL);
7527
7528         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7529             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7530                 return (EINVAL);
7531
7532         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7533         if (rc)
7534                 return (rc);
7535
7536         if (sc->flags & FW_OK) {
7537                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7538                     &cntxt->data[0]);
7539                 if (rc == 0)
7540                         goto done;
7541         }
7542
7543         /*
7544          * Read via firmware failed or wasn't even attempted.  Read directly via
7545          * the backdoor.
7546          */
7547         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7548 done:
7549         end_synchronized_op(sc, 0);
7550         return (rc);
7551 }
7552
7553 static int
7554 load_fw(struct adapter *sc, struct t4_data *fw)
7555 {
7556         int rc;
7557         uint8_t *fw_data;
7558
7559         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7560         if (rc)
7561                 return (rc);
7562
7563         if (sc->flags & FULL_INIT_DONE) {
7564                 rc = EBUSY;
7565                 goto done;
7566         }
7567
7568         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7569         if (fw_data == NULL) {
7570                 rc = ENOMEM;
7571                 goto done;
7572         }
7573
7574         rc = copyin(fw->data, fw_data, fw->len);
7575         if (rc == 0)
7576                 rc = -t4_load_fw(sc, fw_data, fw->len);
7577
7578         free(fw_data, M_CXGBE);
7579 done:
7580         end_synchronized_op(sc, 0);
7581         return (rc);
7582 }
7583
7584 static int
7585 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7586 {
7587         uint32_t addr, off, remaining, i, n;
7588         uint32_t *buf, *b;
7589         uint32_t mw_base, mw_aperture;
7590         int rc;
7591         uint8_t *dst;
7592
7593         rc = validate_mem_range(sc, mr->addr, mr->len);
7594         if (rc != 0)
7595                 return (rc);
7596
7597         memwin_info(sc, win, &mw_base, &mw_aperture);
7598         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7599         addr = mr->addr;
7600         remaining = mr->len;
7601         dst = (void *)mr->data;
7602
7603         while (remaining) {
7604                 off = position_memwin(sc, win, addr);
7605
7606                 /* number of bytes that we'll copy in the inner loop */
7607                 n = min(remaining, mw_aperture - off);
7608                 for (i = 0; i < n; i += 4)
7609                         *b++ = t4_read_reg(sc, mw_base + off + i);
7610
7611                 rc = copyout(buf, dst, n);
7612                 if (rc != 0)
7613                         break;
7614
7615                 b = buf;
7616                 dst += n;
7617                 remaining -= n;
7618                 addr += n;
7619         }
7620
7621         free(buf, M_CXGBE);
7622         return (rc);
7623 }
7624
7625 static int
7626 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7627 {
7628         int rc;
7629
7630         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7631                 return (EINVAL);
7632
7633         if (i2cd->len > sizeof(i2cd->data))
7634                 return (EFBIG);
7635
7636         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7637         if (rc)
7638                 return (rc);
7639         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7640             i2cd->offset, i2cd->len, &i2cd->data[0]);
7641         end_synchronized_op(sc, 0);
7642
7643         return (rc);
7644 }
7645
7646 static int
7647 in_range(int val, int lo, int hi)
7648 {
7649
7650         return (val < 0 || (val <= hi && val >= lo));
7651 }
7652
7653 static int
7654 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7655 {
7656         int fw_subcmd, fw_type, rc;
7657
7658         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7659         if (rc)
7660                 return (rc);
7661
7662         if (!(sc->flags & FULL_INIT_DONE)) {
7663                 rc = EAGAIN;
7664                 goto done;
7665         }
7666
7667         /*
7668          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7669          * sub-command and type are in common locations.)
7670          */
7671         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7672                 fw_subcmd = FW_SCHED_SC_CONFIG;
7673         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7674                 fw_subcmd = FW_SCHED_SC_PARAMS;
7675         else {
7676                 rc = EINVAL;
7677                 goto done;
7678         }
7679         if (p->type == SCHED_CLASS_TYPE_PACKET)
7680                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7681         else {
7682                 rc = EINVAL;
7683                 goto done;
7684         }
7685
7686         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7687                 /* Vet our parameters ..*/
7688                 if (p->u.config.minmax < 0) {
7689                         rc = EINVAL;
7690                         goto done;
7691                 }
7692
7693                 /* And pass the request to the firmware ...*/
7694                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7695                 goto done;
7696         }
7697
7698         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7699                 int fw_level;
7700                 int fw_mode;
7701                 int fw_rateunit;
7702                 int fw_ratemode;
7703
7704                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7705                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7706                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7707                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7708                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7709                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7710                 else {
7711                         rc = EINVAL;
7712                         goto done;
7713                 }
7714
7715                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7716                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7717                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7718                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7719                 else {
7720                         rc = EINVAL;
7721                         goto done;
7722                 }
7723
7724                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7725                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7726                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7727                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7728                 else {
7729                         rc = EINVAL;
7730                         goto done;
7731                 }
7732
7733                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7734                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7735                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7736                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7737                 else {
7738                         rc = EINVAL;
7739                         goto done;
7740                 }
7741
7742                 /* Vet our parameters ... */
7743                 if (!in_range(p->u.params.channel, 0, 3) ||
7744                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7745                     !in_range(p->u.params.minrate, 0, 10000000) ||
7746                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7747                     !in_range(p->u.params.weight, 0, 100)) {
7748                         rc = ERANGE;
7749                         goto done;
7750                 }
7751
7752                 /*
7753                  * Translate any unset parameters into the firmware's
7754                  * nomenclature and/or fail the call if the parameters
7755                  * are required ...
7756                  */
7757                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7758                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762                 if (p->u.params.minrate < 0)
7763                         p->u.params.minrate = 0;
7764                 if (p->u.params.maxrate < 0) {
7765                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7766                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7767                                 rc = EINVAL;
7768                                 goto done;
7769                         } else
7770                                 p->u.params.maxrate = 0;
7771                 }
7772                 if (p->u.params.weight < 0) {
7773                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7774                                 rc = EINVAL;
7775                                 goto done;
7776                         } else
7777                                 p->u.params.weight = 0;
7778                 }
7779                 if (p->u.params.pktsize < 0) {
7780                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7781                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7782                                 rc = EINVAL;
7783                                 goto done;
7784                         } else
7785                                 p->u.params.pktsize = 0;
7786                 }
7787
7788                 /* See what the firmware thinks of the request ... */
7789                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7790                     fw_rateunit, fw_ratemode, p->u.params.channel,
7791                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7792                     p->u.params.weight, p->u.params.pktsize, 1);
7793                 goto done;
7794         }
7795
7796         rc = EINVAL;
7797 done:
7798         end_synchronized_op(sc, 0);
7799         return (rc);
7800 }
7801
7802 static int
7803 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7804 {
7805         struct port_info *pi = NULL;
7806         struct sge_txq *txq;
7807         uint32_t fw_mnem, fw_queue, fw_class;
7808         int i, rc;
7809
7810         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7811         if (rc)
7812                 return (rc);
7813
7814         if (!(sc->flags & FULL_INIT_DONE)) {
7815                 rc = EAGAIN;
7816                 goto done;
7817         }
7818
7819         if (p->port >= sc->params.nports) {
7820                 rc = EINVAL;
7821                 goto done;
7822         }
7823
7824         pi = sc->port[p->port];
7825         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7826                 rc = EINVAL;
7827                 goto done;
7828         }
7829
7830         /*
7831          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7832          * Scheduling Class in this case).
7833          */
7834         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7835             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7836         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7837
7838         /*
7839          * If op.queue is non-negative, then we're only changing the scheduling
7840          * on a single specified TX queue.
7841          */
7842         if (p->queue >= 0) {
7843                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7844                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7845                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7846                     &fw_class);
7847                 goto done;
7848         }
7849
7850         /*
7851          * Change the scheduling on all the TX queues for the
7852          * interface.
7853          */
7854         for_each_txq(pi, i, txq) {
7855                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7856                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7857                     &fw_class);
7858                 if (rc)
7859                         goto done;
7860         }
7861
7862         rc = 0;
7863 done:
7864         end_synchronized_op(sc, 0);
7865         return (rc);
7866 }
7867
7868 int
7869 t4_os_find_pci_capability(struct adapter *sc, int cap)
7870 {
7871         int i;
7872
7873         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7874 }
7875
7876 int
7877 t4_os_pci_save_state(struct adapter *sc)
7878 {
7879         device_t dev;
7880         struct pci_devinfo *dinfo;
7881
7882         dev = sc->dev;
7883         dinfo = device_get_ivars(dev);
7884
7885         pci_cfg_save(dev, dinfo, 0);
7886         return (0);
7887 }
7888
7889 int
7890 t4_os_pci_restore_state(struct adapter *sc)
7891 {
7892         device_t dev;
7893         struct pci_devinfo *dinfo;
7894
7895         dev = sc->dev;
7896         dinfo = device_get_ivars(dev);
7897
7898         pci_cfg_restore(dev, dinfo);
7899         return (0);
7900 }
7901
7902 void
7903 t4_os_portmod_changed(const struct adapter *sc, int idx)
7904 {
7905         struct port_info *pi = sc->port[idx];
7906         static const char *mod_str[] = {
7907                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7908         };
7909
7910         build_medialist(pi, &pi->media);
7911 #ifdef DEV_NETMAP
7912         build_medialist(pi, &pi->nm_media);
7913 #endif
7914
7915         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7916                 if_printf(pi->ifp, "transceiver unplugged.\n");
7917         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7918                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7919         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7920                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7921         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7922                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7923                     mod_str[pi->mod_type]);
7924         } else {
7925                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7926                     pi->mod_type);
7927         }
7928 }
7929
7930 void
7931 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7932 {
7933         struct port_info *pi = sc->port[idx];
7934         struct ifnet *ifp = pi->ifp;
7935
7936         if (link_stat) {
7937                 pi->linkdnrc = -1;
7938                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7939                 if_link_state_change(ifp, LINK_STATE_UP);
7940         } else {
7941                 if (reason >= 0)
7942                         pi->linkdnrc = reason;
7943                 if_link_state_change(ifp, LINK_STATE_DOWN);
7944         }
7945 }
7946
7947 void
7948 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7949 {
7950         struct adapter *sc;
7951
7952         sx_slock(&t4_list_lock);
7953         SLIST_FOREACH(sc, &t4_list, link) {
7954                 /*
7955                  * func should not make any assumptions about what state sc is
7956                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7957                  */
7958                 func(sc, arg);
7959         }
7960         sx_sunlock(&t4_list_lock);
7961 }
7962
7963 static int
7964 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7965 {
7966        return (0);
7967 }
7968
7969 static int
7970 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7971 {
7972        return (0);
7973 }
7974
7975 static int
7976 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7977     struct thread *td)
7978 {
7979         int rc;
7980         struct adapter *sc = dev->si_drv1;
7981
7982         rc = priv_check(td, PRIV_DRIVER);
7983         if (rc != 0)
7984                 return (rc);
7985
7986         switch (cmd) {
7987         case CHELSIO_T4_GETREG: {
7988                 struct t4_reg *edata = (struct t4_reg *)data;
7989
7990                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7991                         return (EFAULT);
7992
7993                 if (edata->size == 4)
7994                         edata->val = t4_read_reg(sc, edata->addr);
7995                 else if (edata->size == 8)
7996                         edata->val = t4_read_reg64(sc, edata->addr);
7997                 else
7998                         return (EINVAL);
7999
8000                 break;
8001         }
8002         case CHELSIO_T4_SETREG: {
8003                 struct t4_reg *edata = (struct t4_reg *)data;
8004
8005                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8006                         return (EFAULT);
8007
8008                 if (edata->size == 4) {
8009                         if (edata->val & 0xffffffff00000000)
8010                                 return (EINVAL);
8011                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8012                 } else if (edata->size == 8)
8013                         t4_write_reg64(sc, edata->addr, edata->val);
8014                 else
8015                         return (EINVAL);
8016                 break;
8017         }
8018         case CHELSIO_T4_REGDUMP: {
8019                 struct t4_regdump *regs = (struct t4_regdump *)data;
8020                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8021                 uint8_t *buf;
8022
8023                 if (regs->len < reglen) {
8024                         regs->len = reglen; /* hint to the caller */
8025                         return (ENOBUFS);
8026                 }
8027
8028                 regs->len = reglen;
8029                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8030                 t4_get_regs(sc, regs, buf);
8031                 rc = copyout(buf, regs->data, reglen);
8032                 free(buf, M_CXGBE);
8033                 break;
8034         }
8035         case CHELSIO_T4_GET_FILTER_MODE:
8036                 rc = get_filter_mode(sc, (uint32_t *)data);
8037                 break;
8038         case CHELSIO_T4_SET_FILTER_MODE:
8039                 rc = set_filter_mode(sc, *(uint32_t *)data);
8040                 break;
8041         case CHELSIO_T4_GET_FILTER:
8042                 rc = get_filter(sc, (struct t4_filter *)data);
8043                 break;
8044         case CHELSIO_T4_SET_FILTER:
8045                 rc = set_filter(sc, (struct t4_filter *)data);
8046                 break;
8047         case CHELSIO_T4_DEL_FILTER:
8048                 rc = del_filter(sc, (struct t4_filter *)data);
8049                 break;
8050         case CHELSIO_T4_GET_SGE_CONTEXT:
8051                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8052                 break;
8053         case CHELSIO_T4_LOAD_FW:
8054                 rc = load_fw(sc, (struct t4_data *)data);
8055                 break;
8056         case CHELSIO_T4_GET_MEM:
8057                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8058                 break;
8059         case CHELSIO_T4_GET_I2C:
8060                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8061                 break;
8062         case CHELSIO_T4_CLEAR_STATS: {
8063                 int i;
8064                 u_int port_id = *(uint32_t *)data;
8065                 struct port_info *pi;
8066
8067                 if (port_id >= sc->params.nports)
8068                         return (EINVAL);
8069                 pi = sc->port[port_id];
8070
8071                 /* MAC stats */
8072                 t4_clr_port_stats(sc, pi->tx_chan);
8073                 pi->tx_parse_error = 0;
8074
8075                 if (pi->flags & PORT_INIT_DONE) {
8076                         struct sge_rxq *rxq;
8077                         struct sge_txq *txq;
8078                         struct sge_wrq *wrq;
8079
8080                         for_each_rxq(pi, i, rxq) {
8081 #if defined(INET) || defined(INET6)
8082                                 rxq->lro.lro_queued = 0;
8083                                 rxq->lro.lro_flushed = 0;
8084 #endif
8085                                 rxq->rxcsum = 0;
8086                                 rxq->vlan_extraction = 0;
8087                         }
8088
8089                         for_each_txq(pi, i, txq) {
8090                                 txq->txcsum = 0;
8091                                 txq->tso_wrs = 0;
8092                                 txq->vlan_insertion = 0;
8093                                 txq->imm_wrs = 0;
8094                                 txq->sgl_wrs = 0;
8095                                 txq->txpkt_wrs = 0;
8096                                 txq->txpkts0_wrs = 0;
8097                                 txq->txpkts1_wrs = 0;
8098                                 txq->txpkts0_pkts = 0;
8099                                 txq->txpkts1_pkts = 0;
8100                                 mp_ring_reset_stats(txq->r);
8101                         }
8102
8103 #ifdef TCP_OFFLOAD
8104                         /* nothing to clear for each ofld_rxq */
8105
8106                         for_each_ofld_txq(pi, i, wrq) {
8107                                 wrq->tx_wrs_direct = 0;
8108                                 wrq->tx_wrs_copied = 0;
8109                         }
8110 #endif
8111                         wrq = &sc->sge.ctrlq[pi->port_id];
8112                         wrq->tx_wrs_direct = 0;
8113                         wrq->tx_wrs_copied = 0;
8114                 }
8115                 break;
8116         }
8117         case CHELSIO_T4_SCHED_CLASS:
8118                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8119                 break;
8120         case CHELSIO_T4_SCHED_QUEUE:
8121                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8122                 break;
8123         case CHELSIO_T4_GET_TRACER:
8124                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8125                 break;
8126         case CHELSIO_T4_SET_TRACER:
8127                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8128                 break;
8129         default:
8130                 rc = EINVAL;
8131         }
8132
8133         return (rc);
8134 }
8135
8136 #ifdef TCP_OFFLOAD
8137 void
8138 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8139     const unsigned int *pgsz_order)
8140 {
8141         struct port_info *pi = ifp->if_softc;
8142         struct adapter *sc = pi->adapter;
8143
8144         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8145         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8146                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8147                 V_HPZ3(pgsz_order[3]));
8148 }
8149
8150 static int
8151 toe_capability(struct port_info *pi, int enable)
8152 {
8153         int rc;
8154         struct adapter *sc = pi->adapter;
8155
8156         ASSERT_SYNCHRONIZED_OP(sc);
8157
8158         if (!is_offload(sc))
8159                 return (ENODEV);
8160
8161         if (enable) {
8162                 /*
8163                  * We need the port's queues around so that we're able to send
8164                  * and receive CPLs to/from the TOE even if the ifnet for this
8165                  * port has never been UP'd administratively.
8166                  */
8167                 if (!(pi->flags & PORT_INIT_DONE)) {
8168                         rc = cxgbe_init_synchronized(pi);
8169                         if (rc)
8170                                 return (rc);
8171                 }
8172
8173                 if (isset(&sc->offload_map, pi->port_id))
8174                         return (0);
8175
8176                 if (!uld_active(sc, ULD_TOM)) {
8177                         rc = t4_activate_uld(sc, ULD_TOM);
8178                         if (rc == EAGAIN) {
8179                                 log(LOG_WARNING,
8180                                     "You must kldload t4_tom.ko before trying "
8181                                     "to enable TOE on a cxgbe interface.\n");
8182                         }
8183                         if (rc != 0)
8184                                 return (rc);
8185                         KASSERT(sc->tom_softc != NULL,
8186                             ("%s: TOM activated but softc NULL", __func__));
8187                         KASSERT(uld_active(sc, ULD_TOM),
8188                             ("%s: TOM activated but flag not set", __func__));
8189                 }
8190
8191                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8192                 if (!uld_active(sc, ULD_IWARP))
8193                         (void) t4_activate_uld(sc, ULD_IWARP);
8194                 if (!uld_active(sc, ULD_ISCSI))
8195                         (void) t4_activate_uld(sc, ULD_ISCSI);
8196
8197                 setbit(&sc->offload_map, pi->port_id);
8198         } else {
8199                 if (!isset(&sc->offload_map, pi->port_id))
8200                         return (0);
8201
8202                 KASSERT(uld_active(sc, ULD_TOM),
8203                     ("%s: TOM never initialized?", __func__));
8204                 clrbit(&sc->offload_map, pi->port_id);
8205         }
8206
8207         return (0);
8208 }
8209
8210 /*
8211  * Add an upper layer driver to the global list.
8212  */
8213 int
8214 t4_register_uld(struct uld_info *ui)
8215 {
8216         int rc = 0;
8217         struct uld_info *u;
8218
8219         sx_xlock(&t4_uld_list_lock);
8220         SLIST_FOREACH(u, &t4_uld_list, link) {
8221             if (u->uld_id == ui->uld_id) {
8222                     rc = EEXIST;
8223                     goto done;
8224             }
8225         }
8226
8227         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8228         ui->refcount = 0;
8229 done:
8230         sx_xunlock(&t4_uld_list_lock);
8231         return (rc);
8232 }
8233
8234 int
8235 t4_unregister_uld(struct uld_info *ui)
8236 {
8237         int rc = EINVAL;
8238         struct uld_info *u;
8239
8240         sx_xlock(&t4_uld_list_lock);
8241
8242         SLIST_FOREACH(u, &t4_uld_list, link) {
8243             if (u == ui) {
8244                     if (ui->refcount > 0) {
8245                             rc = EBUSY;
8246                             goto done;
8247                     }
8248
8249                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8250                     rc = 0;
8251                     goto done;
8252             }
8253         }
8254 done:
8255         sx_xunlock(&t4_uld_list_lock);
8256         return (rc);
8257 }
8258
8259 int
8260 t4_activate_uld(struct adapter *sc, int id)
8261 {
8262         int rc;
8263         struct uld_info *ui;
8264
8265         ASSERT_SYNCHRONIZED_OP(sc);
8266
8267         if (id < 0 || id > ULD_MAX)
8268                 return (EINVAL);
8269         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8270
8271         sx_slock(&t4_uld_list_lock);
8272
8273         SLIST_FOREACH(ui, &t4_uld_list, link) {
8274                 if (ui->uld_id == id) {
8275                         if (!(sc->flags & FULL_INIT_DONE)) {
8276                                 rc = adapter_full_init(sc);
8277                                 if (rc != 0)
8278                                         break;
8279                         }
8280
8281                         rc = ui->activate(sc);
8282                         if (rc == 0) {
8283                                 setbit(&sc->active_ulds, id);
8284                                 ui->refcount++;
8285                         }
8286                         break;
8287                 }
8288         }
8289
8290         sx_sunlock(&t4_uld_list_lock);
8291
8292         return (rc);
8293 }
8294
8295 int
8296 t4_deactivate_uld(struct adapter *sc, int id)
8297 {
8298         int rc;
8299         struct uld_info *ui;
8300
8301         ASSERT_SYNCHRONIZED_OP(sc);
8302
8303         if (id < 0 || id > ULD_MAX)
8304                 return (EINVAL);
8305         rc = ENXIO;
8306
8307         sx_slock(&t4_uld_list_lock);
8308
8309         SLIST_FOREACH(ui, &t4_uld_list, link) {
8310                 if (ui->uld_id == id) {
8311                         rc = ui->deactivate(sc);
8312                         if (rc == 0) {
8313                                 clrbit(&sc->active_ulds, id);
8314                                 ui->refcount--;
8315                         }
8316                         break;
8317                 }
8318         }
8319
8320         sx_sunlock(&t4_uld_list_lock);
8321
8322         return (rc);
8323 }
8324
8325 int
8326 uld_active(struct adapter *sc, int uld_id)
8327 {
8328
8329         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8330
8331         return (isset(&sc->active_ulds, uld_id));
8332 }
8333 #endif
8334
8335 /*
8336  * Come up with reasonable defaults for some of the tunables, provided they're
8337  * not set by the user (in which case we'll use the values as is).
8338  */
8339 static void
8340 tweak_tunables(void)
8341 {
8342         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8343
8344         if (t4_ntxq10g < 1)
8345                 t4_ntxq10g = min(nc, NTXQ_10G);
8346
8347         if (t4_ntxq1g < 1)
8348                 t4_ntxq1g = min(nc, NTXQ_1G);
8349
8350         if (t4_nrxq10g < 1)
8351                 t4_nrxq10g = min(nc, NRXQ_10G);
8352
8353         if (t4_nrxq1g < 1)
8354                 t4_nrxq1g = min(nc, NRXQ_1G);
8355
8356 #ifdef TCP_OFFLOAD
8357         if (t4_nofldtxq10g < 1)
8358                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8359
8360         if (t4_nofldtxq1g < 1)
8361                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8362
8363         if (t4_nofldrxq10g < 1)
8364                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8365
8366         if (t4_nofldrxq1g < 1)
8367                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8368
8369         if (t4_toecaps_allowed == -1)
8370                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8371 #else
8372         if (t4_toecaps_allowed == -1)
8373                 t4_toecaps_allowed = 0;
8374 #endif
8375
8376 #ifdef DEV_NETMAP
8377         if (t4_nnmtxq10g < 1)
8378                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8379
8380         if (t4_nnmtxq1g < 1)
8381                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8382
8383         if (t4_nnmrxq10g < 1)
8384                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8385
8386         if (t4_nnmrxq1g < 1)
8387                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8388 #endif
8389
8390         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8391                 t4_tmr_idx_10g = TMR_IDX_10G;
8392
8393         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8394                 t4_pktc_idx_10g = PKTC_IDX_10G;
8395
8396         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8397                 t4_tmr_idx_1g = TMR_IDX_1G;
8398
8399         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8400                 t4_pktc_idx_1g = PKTC_IDX_1G;
8401
8402         if (t4_qsize_txq < 128)
8403                 t4_qsize_txq = 128;
8404
8405         if (t4_qsize_rxq < 128)
8406                 t4_qsize_rxq = 128;
8407         while (t4_qsize_rxq & 7)
8408                 t4_qsize_rxq++;
8409
8410         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8411 }
8412
8413 static struct sx mlu;   /* mod load unload */
8414 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8415
8416 static int
8417 mod_event(module_t mod, int cmd, void *arg)
8418 {
8419         int rc = 0;
8420         static int loaded = 0;
8421
8422         switch (cmd) {
8423         case MOD_LOAD:
8424                 sx_xlock(&mlu);
8425                 if (loaded++ == 0) {
8426                         t4_sge_modload();
8427                         sx_init(&t4_list_lock, "T4/T5 adapters");
8428                         SLIST_INIT(&t4_list);
8429 #ifdef TCP_OFFLOAD
8430                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8431                         SLIST_INIT(&t4_uld_list);
8432 #endif
8433                         t4_tracer_modload();
8434                         tweak_tunables();
8435                 }
8436                 sx_xunlock(&mlu);
8437                 break;
8438
8439         case MOD_UNLOAD:
8440                 sx_xlock(&mlu);
8441                 if (--loaded == 0) {
8442                         int tries;
8443
8444                         sx_slock(&t4_list_lock);
8445                         if (!SLIST_EMPTY(&t4_list)) {
8446                                 rc = EBUSY;
8447                                 sx_sunlock(&t4_list_lock);
8448                                 goto done_unload;
8449                         }
8450 #ifdef TCP_OFFLOAD
8451                         sx_slock(&t4_uld_list_lock);
8452                         if (!SLIST_EMPTY(&t4_uld_list)) {
8453                                 rc = EBUSY;
8454                                 sx_sunlock(&t4_uld_list_lock);
8455                                 sx_sunlock(&t4_list_lock);
8456                                 goto done_unload;
8457                         }
8458 #endif
8459                         tries = 0;
8460                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8461                                 uprintf("%ju clusters with custom free routine "
8462                                     "still is use.\n", t4_sge_extfree_refs());
8463                                 pause("t4unload", 2 * hz);
8464                         }
8465 #ifdef TCP_OFFLOAD
8466                         sx_sunlock(&t4_uld_list_lock);
8467 #endif
8468                         sx_sunlock(&t4_list_lock);
8469
8470                         if (t4_sge_extfree_refs() == 0) {
8471                                 t4_tracer_modunload();
8472 #ifdef TCP_OFFLOAD
8473                                 sx_destroy(&t4_uld_list_lock);
8474 #endif
8475                                 sx_destroy(&t4_list_lock);
8476                                 t4_sge_modunload();
8477                                 loaded = 0;
8478                         } else {
8479                                 rc = EBUSY;
8480                                 loaded++;       /* undo earlier decrement */
8481                         }
8482                 }
8483 done_unload:
8484                 sx_xunlock(&mlu);
8485                 break;
8486         }
8487
8488         return (rc);
8489 }
8490
8491 static devclass_t t4_devclass, t5_devclass;
8492 static devclass_t cxgbe_devclass, cxl_devclass;
8493
8494 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8495 MODULE_VERSION(t4nex, 1);
8496 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8497
8498 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8499 MODULE_VERSION(t5nex, 1);
8500 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8501
8502 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8503 MODULE_VERSION(cxgbe, 1);
8504
8505 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8506 MODULE_VERSION(cxl, 1);